<?php
/**
* 指定したプロセス数で並列処理を実行する
*
* @param array $url_list URLの配列
* @param boolean $url_as_key 結果配列を返すときに、urlをキーにする
* @param int $timeout タイムアウト秒数 0だと無制限
* @return array 結果配列
*/
function execute($url_list, $url_as_key = false, $timeout=0) {
// set your process number
$process = 5;
$is_over_process = false;
if ($process < count($url_list)) {
// chunk url list / process number*
$url_chunk = array_chunk($url_list, $process);
$is_over_process = true;
}
$ret = array();
if ($is_over_process && !empty($url_chunk)) {
foreach ($url_chunk as $key => $url_list) {
echo "chunk start:{$key}\n";
$res = fetch_multi_url($url_list, $url_as_key, $timeout);
if (!empty($res)) {
$ret = array_merge($ret, $res);
} else {
continue;
}
}
} else if (!$is_over_process && !empty($url_list)){
$ret = fetch_multi_url($url_list, $url_as_key, $timeout);
} else {
echo "url invalid::";
}
return $ret;
}
/**
* curl_multi_execの並列処理
* ほぼboilerplate
*
* @param array $url_list URLの配列
* @param boolean $url_as_key 結果配列を返すときに、urlをキーにする
* @param int $timeout タイムアウト秒数 0だと無制限
* @return array 結果配列
*/
function fetch_multi_url($url_list, $url_as_key, $timeout) {
$mh = curl_multi_init();
foreach ($url_list as $i => $url) {
$ch[$i] = curl_init($url);
curl_setopt($ch[$i],CURLOPT_RETURNTRANSFER,1);
//タイムアウト
if ($timeout){
curl_setopt($ch[$i],CURLOPT_TIMEOUT,$timeout);
}
curl_multi_add_handle($mh,$ch[$i]);
}
//URLを取得
//すべて取得するまでループ
$active = null;
do {
$mrc = curl_multi_exec($mh,$active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
while ($active and $mrc == CURLM_OK) {
if (curl_multi_select($mh) != -1) {
do {
$mrc = curl_multi_exec($mh,$active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
}
}
if ($mrc != CURLM_OK) {
echo '読み込みエラーが発生しました:'.$mrc;
}
//ソースコードを取得
$res = array();
foreach ($url_list as $i => $url) {
if (($err = curl_error($ch[$i])) == '') {
// url_as_keyがtrueの場合、urlをキーとして格納
if ($url_as_key) {
$res[$url] = curl_multi_getcontent($ch[$i]);
// そうでない場合は、ただ配列に入れる
} else {
$res[$i] = curl_multi_getcontent($ch[$i]);
}
} else {
echo '取得に失敗しました:'.$url_list[$i].'<br />';
}
curl_multi_remove_handle($mh,$ch[$i]);
curl_close($ch[$i]);
}
curl_multi_close($mh);
return $res;
}
// 並列実行したいurl list
$url_list = array(
"http://www.kinopyo.com/blog/ipad-2-not-charging-when-connected-to-pc-usb",
"http://www.kinopyo.com/blog/the-first-app-i-installed-to-ipad2",
"http://www.kinopyo.com/blog/chrome-warn-before-quitting-with-command-q-in-mac",
"http://www.kinopyo.com/blog/reply-to-all-always-in-gmail",
"http://www.kinopyo.com/blog/lion-fullscreen-shortcut-key-conflict-with-evernote-client",
"http://www.kinopyo.com/blog/how-to-set-gesture-for-chrome-to-swipe-back-and-forth-in-lion",
"http://www.kinopyo.com/blog/3-free-ebooks-for-study-coffeescript"
);
// start time
$start_time = microtime(true);
// execute
$res = execute($url_list, true);
// execute time
$time = microtime(true) - $start_time;
// play with the result
// here I just get the page title
$titles = array();
foreach ($res as $url => $html) {
preg_match('{<title>(.*)</title>}',$html, $match_title);
$titles[$url] = $match_title[1];
}
echo "Result:\n";
echo "time:{$time} sec\n";
print_r($titles);
