利用curl，正则表达式做的一个php蜘蛛抓取器

php中文网

发布时间：2016-07-25 09:08:53

1220人浏览过

来源于php中文网

原创

凤网fcms内容管理系统
get.php 抓取框架，对网页内容的分析处理并进行相关替换
std.php 通用正则
news_67_com.php 对http://news.67.com 的抓取分析器
先抓列表，再抓内容页。
还欠缺监控，统计，错误处理功能。个人觉得还是比较好玩。

include_once dirname(__FILE__) . '/std.php';
$site = array(
'aname' => '中国娱乐网',
'domain' => 'news.67.com',
'dirname' => '目录名称，用于匹配基于目录不同的正文',
'gettype' => 'default',
//获取主文件
'creg' => '/(?si)(.*?)\/',
'code' => 'utf-8',
'sub' => '获取子目录正则',
'content' => 'tag1',
'img_upload'=> array('tag1' => ''),
//下一页
'reg_next' => '/(?is)下一页\>\>/',
'key0' => '/(?is)/',
'key0_ap' => array(array(',', '|'), ' '),
'tag0' => '/(?is)
([^]*?)/',
'tag0_arp' => array(
array(
'/(?is)$组图$/',
'/(?is)$图$/',
'/(?is)\(图\.\./',
'/(?is)\(组图\.\./',
'/(?is)\./',
'/(?is)(《|》)/',
),
array(
'', '', '', '', '', '',
)
),
'tag1' => '/(?is)
(.*?)/',
'tag1_brp' => array(
array(
'/(?is)（.*?）/',
'/(?is)$.*?$/',
'/(?is)\s*
.*?(.*?)\s*/',
'/(?is)\s*
\s*/',

AI Home Tab
把你喜欢的AI放到首页

下载
'/(?is)\s*
\s*/',
'/(?is)　/',
'/(?is)
/',
'/(?is)\s*
\s*/',
'/(?is)\s*
\s*/',
'/(?is)\s*\s*/',
'/(?is)\s*\s*/',
'/(?is)\s*
\s*/',
),
array(
'', '', '
', '
', '

', '', '', '

', '

', '

', '
', '
'
),
),
'tag1_arp' => array(
array(
'/(?is)
/',
'/(?is)/'
),
array(
'', ''
),
),
'strip' => array('tag1' => ''),
'tag2' => '/(?is)
导读：\s*(.*?)\s*/',
'tag2_arp' => array(
array(
'/(?is)　/'
),
array(
''
),
),
'tag3' => '/(?is)(中国娱乐网)/',
'tag4' => '/(?is)
日期：(\d+-\d+-\d+ \d+:\d+:\d+).*?/',
);
$map = array(
'tag' => 'key0',
'title' => 'tag0',
'content' => 'tag1',
'summary' => 'tag2',
'source' => 'tag3',
'pub_date' => 'tag4',
);
$site_list = array(
'aname' => '中国娱乐网',
'domain' => 'www.67.com',
'gettype' => 'default',
'creg' => '/(?si)
(.*?)
/',
'code' => 'gbk',
'reg_next' => '/(?si)
下一页/',
//链接
'tag0' => '/(?is)
.*?[^]*?.*?/',
//标题

'tag1' => '/(?is)
.*?([^]*?).*?/',
'tag1_arp' => array(

'', '', '', '', '', '',

$site_list_sub = array();

复制代码

global $std;
$std = array(
'url' => '[0-9a-zA-Z\.\:\-\/%_#;&]+',
'img' => '/(?is)/',
);

复制代码

/**
* test.php
*
* @author xzfred
* @copyright 2009 fengone.com
* @created 2010-12-07 .
* @version $Id: php.php 3 2008-10-10 07:49:21Z fred $
* SVNPath $HeadURL: http://192.168.0.16/svn/vim/skeletons/php.php $
*/
/*
include_once "std.php";
include_once "lady_163_com.php";
*/
include_once $GLOBALS['g_dir_core'] . "get.php";
//================================================================================
include_once DIR_HOST_TAG . '/tuku_ent_china_com.php';
$obj = new FcHtmlParse($site);
$c = $obj->parse(file_get_contents("http://tuku.ent.china.com/fun/html/2011-08-23/181703.xml"));
echo "\n\n\n ===================\n";
echo $c['field']['tag1'][0];
echo "\n\n\n ===================\n";
var_dump($c);
exit();
//列表测试
$obj = new FcHtmlParse($site_list);
$c = $obj->parse(file_get_contents("http://tuku.ent.china.com/fun/html/3569_1.html"));
var_dump($c);
exit();
/*
$obj = new FcHtmlGet($site);
$c = $obj->getPage('http://star.pclady.com.cn/entertainment/ss/1106/703240.html');
var_dump($c);
$obj = new FcHtmlGet($site);
$c = $obj->getPage('http://star.pclady.com.cn/entertainment/ss/1106/703240.html');
var_dump($c);
$obj = new FcHtmlParse($site);
$img_obj = new FcHtmlImgUpload($site);
$data = file_get_contents("e:/b.html");
$c = $obj->parse($data);
$ic = $img_obj->upload($c['tag']['tag1'][0]);
var_dump($ic);
$data = file_get_contents("e:/a.html");
$c = $obj->parse($data);
$ic = $img_obj->upload($c['tag']['tag1'][0]);
var_dump($ic);
*/
//var_dump($c['tag']['tag1']);