Kinopyo Blog

<?php
/**
* 指定したプロセス数で並列処理を実行する
*
* @param array $url_list URLの配列
* @param boolean $url_as_key 結果配列を返すときに、urlをキーにする
* @param int $timeout タイムアウト秒数 0だと無制限
* @return array 結果配列
*/
function execute($url_list, $url_as_key = false, $timeout=0) {
    // set your process number
    $process = 5;

    $is_over_process = false;
    if ($process < count($url_list)) {
        // chunk url list / process number*
        $url_chunk = array_chunk($url_list, $process);
        $is_over_process = true;
    }

    $ret = array(); 
        
    if ($is_over_process && !empty($url_chunk)) {

        foreach ($url_chunk as $key => $url_list) {
            echo "chunk start:{$key}\n";
            
            $res = fetch_multi_url($url_list, $url_as_key, $timeout);
            if (!empty($res)) {
                $ret = array_merge($ret, $res);
            } else {
                continue;
            }

        }
    } else if (!$is_over_process && !empty($url_list)){
        $ret = fetch_multi_url($url_list, $url_as_key, $timeout);
    } else {
        echo "url invalid::";
    }
    
    return $ret;
    
}

/**
 * curl_multi_execの並列処理
 * ほぼboilerplate
 *
* @param array $url_list URLの配列
* @param boolean $url_as_key 結果配列を返すときに、urlをキーにする
* @param int $timeout タイムアウト秒数 0だと無制限
* @return array 結果配列
 */
function fetch_multi_url($url_list, $url_as_key, $timeout) {
    $mh = curl_multi_init();
    foreach ($url_list as $i => $url) {
        $ch[$i] = curl_init($url);
        curl_setopt($ch[$i],CURLOPT_RETURNTRANSFER,1);

        //タイムアウト
        if ($timeout){
            curl_setopt($ch[$i],CURLOPT_TIMEOUT,$timeout);
        }

        curl_multi_add_handle($mh,$ch[$i]);
    }

    //URLを取得
    //すべて取得するまでループ
    $active = null;
    do {
        $mrc = curl_multi_exec($mh,$active);
    } while ($mrc == CURLM_CALL_MULTI_PERFORM);

    while ($active and $mrc == CURLM_OK) {
        if (curl_multi_select($mh) != -1) {
            do {
                $mrc = curl_multi_exec($mh,$active);
            } while ($mrc == CURLM_CALL_MULTI_PERFORM);
        }
    }

    if ($mrc != CURLM_OK) {
        echo '読み込みエラーが発生しました:'.$mrc;
    }

    //ソースコードを取得
    $res = array();
    foreach ($url_list as $i => $url) {
        if (($err = curl_error($ch[$i])) == '') {
            // url_as_keyがtrueの場合、urlをキーとして格納
            if ($url_as_key) {
                $res[$url] = curl_multi_getcontent($ch[$i]);
            // そうでない場合は、ただ配列に入れる
            } else {
                $res[$i] = curl_multi_getcontent($ch[$i]);
            }
        } else {
            echo '取得に失敗しました:'.$url_list[$i].'<br />';
        }
        curl_multi_remove_handle($mh,$ch[$i]);
        curl_close($ch[$i]);
    }
    curl_multi_close($mh);

    return $res;
}


// 並列実行したいurl list
$url_list = array(
    "http://www.kinopyo.com/blog/ipad-2-not-charging-when-connected-to-pc-usb",
    "http://www.kinopyo.com/blog/the-first-app-i-installed-to-ipad2",
    "http://www.kinopyo.com/blog/chrome-warn-before-quitting-with-command-q-in-mac",
    "http://www.kinopyo.com/blog/reply-to-all-always-in-gmail",
    "http://www.kinopyo.com/blog/lion-fullscreen-shortcut-key-conflict-with-evernote-client",
    "http://www.kinopyo.com/blog/how-to-set-gesture-for-chrome-to-swipe-back-and-forth-in-lion",
    "http://www.kinopyo.com/blog/3-free-ebooks-for-study-coffeescript"
);

// start time
$start_time = microtime(true);

// execute
$res = execute($url_list, true);

// execute time
$time = microtime(true) - $start_time;

// play with the result
// here I just get the page title
$titles = array();
foreach ($res as $url => $html) {
    preg_match('{<title>(.*)</title>}',$html, $match_title);
    $titles[$url] = $match_title[1];
}

echo "Result:\n";
echo "time:{$time} sec\n";
print_r($titles);