代码片段 · 2015年5月10日

PHP curl的多线程操作

许多人对php手册中语焉不详的curl_multi一族的函数头疼不已,它们文档少,给的例子 更是简单的让你无从借鉴,我也曾经找了许多网页,都没见一个完整的应用例子。
# curl_multi_add_handle
# curl_multi_close
# curl_multi_exec
# curl_multi_getcontent
# curl_multi_info_read
# curl_multi_init
# curl_multi_remove_handle
# curl_multi_select

一般来说,想到要用这些函数时,目的显然应该是要同时请求多个url,而不是一个一个依次请求,否则不如自己循环去调curl_exec好了。

步骤总结如下:

第一步:调用curl_multi_init
第二步:循环调用curl_multi_add_handle
这一步需要注意的是,curl_multi_add_handle的第二个参数是由curl_init而来的子handle。
第三步:持续调用curl_multi_exec
第四步:根据需要循环调用curl_multi_getcontent获取结果
第五步:调用curl_multi_remove_handle,并为每个字handle调用curl_close
第六步:调用curl_multi_close

这里有一个网上找的简单例子,其作者称为dirty的例子,(稍后我会说明为何dirty):
/*
Here’s a quick and dirty example for curl-multi from PHP, tested on PHP 5.0.0RC1 CLI / FreeBSD 5.2.1
*/

$connomains = array(
“http://www.cnn.com/”,
“http://www.canada.com/”,
“http://www.yahoo.com/”
);

$mh = curl_multi_init();

foreach ($connomains as $i => $url) {
$conn[$i]=curl_init($url);
curl_setopt($conn[$i],CURLOPT_RETURNTRANSFER,1);
curl_multi_add_handle ($mh,$conn[$i]);
}

do { $n=curl_multi_exec($mh,$active); } while ($active);

foreach ($connomains as $i => $url) {
$res[$i]=curl_multi_getcontent($conn[$i]);
curl_close($conn[$i]);
}

print_r($res);

整个使用过程差不多就是这样,但是,这个简单代码有个致命弱点,就是在do循环的那段,在整个url请求期间是个死循环,它会轻易导致CPU占用100%。

现在我们来改进它,这里要用到一个几乎没有任何文档的函数curl_multi_select了,虽然C的curl库对select有说明,但是,php里的接口和用法确与C中有不同。

把上面do的那段改成下面这样:
do {
$mrc = curl_multi_exec($mh,$active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
while ($active and $mrc == CURLM_OK) {
if (curl_multi_select($mh) != -1) {
do {
$mrc = curl_multi_exec($mh, $active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
}
}

因为$active要等全部url数据接受完毕才变成false,所以这里用到了curl_multi_exec的返回值判断是否还有数据,当有数据的时候就不停调用curl_multi_exec,暂时没有数据就进入select阶段,新数据一来就可以被唤醒继续执行。这里的好处就是CPU的无谓消耗没有了。

另外:还有一些细节的地方可能有时候要遇到:

控制每一个请求的超时时间,在curl_multi_add_handle之前通过curl_setopt去做:
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);

判断是否超时了或者其他错误,在curl_multi_getcontent之前用:curl_error($conn[$i]);

孤魂写的一个多线程的函数版:

01
02
03
04
05
06
07
08
09
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
<?php
$text = remote(array('http://www.kalvin.cn/about-me/','http://www.kalvin.cn/'));
print_r($text);
function remote($urls) {
    if (!is_array($urls) or count($urls) == 0) {
        return false;
    }
    $curl = $text = array();
    $handle = curl_multi_init();
    foreach($urls as $k => $v) {
        $nurl[$k]= preg_replace('~([^:\/\.]+)~ei', "rawurlencode('\\1')", $v);
        $curl[$k] = curl_init($nurl[$k]);
        curl_setopt($curl[$k], CURLOPT_RETURNTRANSFER, 1);
        curl_setopt($curl[$k], CURLOPT_HEADER, 0);
        curl_multi_add_handle ($handle, $curl[$k]);
    }
    $active = null;
    do {
        $mrc = curl_multi_exec($handle, $active);
    } while ($mrc == CURLM_CALL_MULTI_PERFORM);
    while ($active && $mrc == CURLM_OK) {
        if (curl_multi_select($handle) != -1) {
            do {
                $mrc = curl_multi_exec($handle, $active);
            } while ($mrc == CURLM_CALL_MULTI_PERFORM);
        }
    }
    foreach ($curl as $k => $v) {
        if (curl_error($curl[$k]) == "") {
        $text[$k] = (string) curl_multi_getcontent($curl[$k]);
        }
        curl_multi_remove_handle($handle, $curl[$k]);
        curl_close($curl[$k]);
    }
    curl_multi_close($handle);
    return $text;
}

一个多线程的采集类
https://github.com/xfeng/MultiHttpRequest

001
002
003
004
005
006
007
008
009
010
011
012
013
014
015
016
017
018
019
020
021
022
023
024
025
026
027
028
029
030
031
032
033
034
035
036
037
038
039
040
041
042
043
044
045
046
047
048
049
050
051
052
053
054
055
056
057
058
059
060
061
062
063
064
065
066
067
068
069
070
071
072
073
074
075
076
077
078
079
080
081
082
083
084
085
086
087
088
089
090
091
092
093
094
095
096
097
098
099
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
<?php
/*
    PHP通用采集
    支持多列表匹配;
    页面内容压缩后匹配,降低规则难度;
    多线程采集,速度更快,为防止目标站限制注意设置并发数量;
    采集后导出content里的内容和images文件夹中图片即可;
    echo str_pad(1,8,"0",STR_PAD_LEFT);00000001
*/
header("Content-type: text/html;charset=utf-8");
set_time_limit(0);
require("libs/class_curl_multi.php");
//连接数据库
$link = mysql_connect("127.0.0.1:3306","root","");
mysql_select_db("test",$link);
//清空数据库
mysql_query("TRUNCATE TABLE content");
//域名前缀
$base = "http://sellbest.net";
//需要采集的规则列表(分页)
$list = array(
    //'http://sellbest.net/by-brand/page[1-14]/1-PRADA.html',
    'http://sellbest.net/by-brand/limit1800/page[1-1]/1-PRADA.html',
);
//在列表页面内容链接表达式
$list_rules = '<p class="productName"><a href="(.*)">.*</a>';
//内容页面信息字段表达式
$detail_rules = array(
    'meta_title'=>'<title>(.*)</title>',
    'meta_keywords'=>'<meta name="keywords" content="(.*)" />',
    'meta_description'=>'<meta name="description" content="(.*)" />',
    'product_name'=>'<h4 class="h4-title float-l"> (.*)</h4>',
    'product_image'=>'<div class="v-inner">.*<a href="(.*)" id="originalImg"><img src=".*" alt=".*" /></a>.*</div>',
    'product_price'=>'Our Price : <strong>(.*)</strong>',
    'product_description'=>'<div class="description-text" id="description"><div class="border-cont">(.*)</div>',
);
//实例
$mp = new MultiHttpRequest();
//调试使用记录采集条目
$j = 1;
//每次并发几个链接
$limit = 10;
//开始采集
foreach ($list as $link) {
    
    //解析列表页数
    preg_match_all('#\[(.*)\]#isU',$link,$_page);
    if($_page[1][0]==''){
        continue;
    }
    $pages = explode('-',$_page[1][0]);
    if(count($pages) != 2){
        continue;
    }
    
    $urls = array();
    
    for($i=$pages[0];$i< =$pages[1];$i++){
        if(count($urls) < $limit){
            $urls[] = preg_replace('#\[(.*)\]#isU',$i,$link);
            if($i != $pages[1]){
                continue;
            }
        }
        //采集列表内容
        $mp->set_urls($urls);
        $contents = $mp->start();
                
        foreach ($contents as $content) {
            
                    $content = _prefilter($content);
                    //debug
                    //exit($content);
                    
                    //匹配内容
                    preg_match_all('#'.addslashes($list_rules).'#isU',$content,$pregArr);
                    
                    $detail_urls = array();
                    foreach($pregArr[1] as $detail_key=>$detail_value){
                        $data = array();
                        if(count($detail_urls) < $limit ){
                                $detail_urls[] = $base.$detail_value;
                                if($pregArr[1][$detail_key+1] != ''){
                                    continue;
                                }
                        }
                        
                        //print_r($detail_urls);
                        //continue;                    
                        $mp->set_urls($detail_urls);
                        
                        $details = $mp->start();
                        //图片路径临时存放
                        $images_urls = array();
                        
                        //采集内容页面
                        foreach ($details as $detail) {
                            $detail = _prefilter($detail);
                            //debug
                            exit($detail);
                            
                            foreach ($detail_rules as $key => $value) {
                                
                                preg_match_all('#'.addslashes($value).'#isU',$detail,$detailArr);
                                //处理特殊这段信息
                                switch ($key) {
                                    case 'product_image':
                                        $data[$key] = "images/".md5($detailArr[1][0]).".jpg";
                                        if(!file_exists($data[$key])){
                                            $images_urls[$data[$key]] = $base.$detailArr[1][0];
                                            //file_put_contents($data[$key],$mp->get_content($base.$detailArr[1][0]));
                                        }
                                        break;
                                    case 'product_description':
                                        $data[$key] = trim(strip_tags($detailArr[1][0]));
                                        break;
                                    default:
                                        $data[$key] = $detailArr[1][0];
                                        break;
                                }
                                                
                            }
                            
                            
                            //产品url        
                            $data['product_url'] = _title($data['product_name']);
                            //转义采集后的数据
                            foreach ($data as $_k => $_v) {
                                $data[$_k] = addslashes($_v);
                            }
                            //入库
                            $r = mysql_query("
                            insert into content values(
                            null,
                            '{$data['meta_title']}',
                            '{$data['meta_keywords']}',
                            '{$data['meta_description']}',
                            '{$data['product_name']}',
                            '{$data['product_image']}',
                            '{$data['product_price']}',
                            '{$data['product_description']}',
                            '{$data['product_url']}')");
                            //打印log
                            _flush($j++."|".$r."|".$data['product_name']."\n");
                            //_flush($data);
                        }
                        //远程图片本地化
                        $mp->set_urls($images_urls);                    
                        $images = $mp->start();                     
                        foreach ((array)$images as $image_key => $image_value) {
                            _flush($image_key."\n");
                            file_put_contents($image_key,$image_value);
                        }
                        //清空内容url并加入本次循环url。不然本次会被跳过
                        $detail_urls = array($base.$detail_value);
            }
        }
        //清空内容url并加入本次循环url。不然本次会被跳过
        $urls = array(preg_replace('#\[(.*)\]#isU',$i,$link));
    }
}
//=========================================================
function _flush($msg)
{
    print_r ($msg);
    ob_flush();
    flush();
}
function _title($title, $separator = '-' )
{
    $title = preg_replace('![^'.preg_quote($separator).'\pL\pN\s]+!u', '', strtolower($title));
    $title = preg_replace('!['.preg_quote($separator).'\s]+!u', $separator, $title);
    return trim($title, $separator);
}
function _prefilter($output) {
    $output=preg_replace("/\/\/[\S\f\t\v ]*?;[\r|\n]/","",$output);
    $output=preg_replace("/\< \!\-\-[\s\S]*?\-\-\>/","",$output);
    $output=preg_replace("/\>[\s]+\< ",$output);
    $output=preg_replace("/;[\s]+/",";",$output);
    $output=preg_replace("/[\s]+\}/","}",$output);
    $output=preg_replace("/}[\s]+/","}",$output);
    $output=preg_replace("/\{[\s]+/","{",$output);
    $output=preg_replace("/([\s]){2,}/","$1",$output);
    $output=preg_replace("/[\s]+\=[\s]+/","=",$output);
    return $output;
}
?>