百万级别知乎用户数据抓取与分析之PHP开发

  $url = 'http://www.zhihu.com/people/mora-hu/about'; //此处mora-hu代表用户ID
  $ch = curl_init($url); //初始化会话
  curl_setopt($ch,CURLOPT_HEADER,0);
  curl_setopt($ch,CURLOPT_COOKIE,$this->config_arr['user_cookie']); //设置请求COOKIE
  curl_setopt($ch,CURLOPT_USERAGENT,$_SERVER['HTTP_USER_AGENT']);
  curl_setopt($ch,CURLOPT_RETURNTRANSFER,1); //将curl_exec()获取的信息以文件流的形式返回,而不是直接输出.
  curl_setopt($ch,CURLOPT_FOLLOWLOCATION,1); 
  $result = curl_exec($ch);
  return $result; //抓取的结果

function getImg($url,$u_id)
{
  if (file_exists('./images/' . $u_id . ".jpg"))
  {
    return "images/$u_id" . '.jpg';
  }
  if (empty($url))
  {
    return '';
  }
  $context_options = array( 
    'http' => 
    array(
      'header' => "Referer:http://www.zhihu.com"//带上referer参数 
  　　)
　　);
   
  $context = stream_context_create($context_options); 
  $img = file_get_contents('http:' . $url,FALSE,$context);
  file_put_contents('./images/' . $u_id . ".jpg",$img);
  return "images/$u_id" . '.jpg';
}

  $mh = curl_multi_init(); //返回一个新cURL批处理句柄
    for ($i = ; $i < $max_size; $i++)
    {
      $ch = curl_init(); //初始化单个cURL会话
      curl_setopt($ch,);
      curl_setopt($ch,CURLOPT_URL,'http://www.zhihu.com/people/' . $user_list[$i] . '/about');
      curl_setopt($ch,self::$user_cookie);
      curl_setopt($ch,'Mozilla/. (Windows NT .; WOW) AppleWebKit/. (KHTML,like Gecko) Chrome/... Safari/.');
      curl_setopt($ch,true); 
      curl_setopt($ch,);
      $requestMap[$i] = $ch;
      curl_multi_add_handle($mh,$ch); //向curl批处理会话中添加单独的curl句柄
    }
    $user_arr = array();
    do {
            //运行当前 cURL 句柄的子连接
      while (($cme = curl_multi_exec($mh,$active)) == CURLM_CALL_MULTI_PERFORM);
      if ($cme != CURLM_OK) {break;}
            //获取当前解析的cURL的相关传输信息
      while ($done = curl_multi_info_read($mh))
      {
        $info = curl_getinfo($done['handle']);
        $tmp_result = curl_multi_getcontent($done['handle']);
        $error = curl_error($done['handle']);
        $user_arr[] = array_values(getUserInfo($tmp_result));
        //保证同时有$max_size个请求在处理
        if ($i < sizeof($user_list) && isset($user_list[$i]) && $i < count($user_list))
        {
          $ch = curl_init();
          curl_setopt($ch,);
          curl_setopt($ch,'http://www.zhihu.com/people/' . $user_list[$i] . '/about');
          curl_setopt($ch,self::$user_cookie);
          curl_setopt($ch,like Gecko) Chrome/... Safari/.');
          curl_setopt($ch,true); 
          curl_setopt($ch,);
          $requestMap[$i] = $ch;
          curl_multi_add_handle($mh,$ch);
          $i++;
        }
        curl_multi_remove_handle($mh,$done['handle']);
      }
      if ($active)
        curl_multi_select($mh,);
    } while ($active);
    curl_multi_close($mh);
    return $user_arr;  

<?php
  $redis = new Redis();
  $redis->connect('127.0.0.1','6379');
  $redis->set('tmp','value');
  if ($redis->exists('tmp'))
  {
    echo $redis->get('tmp') . "n";
  }

//PHP多进程demo
  //fork10个进程
  for ($i = 0; $i < 10; $i++) {
    $pid = pcntl_fork();
    if ($pid == -1) {
      echo "Could not fork!n";
      exit(1);
    }
    if (!$pid) {
      echo "child process $i runningn";
      //子进程执行完毕之后就退出,以免继续fork出新的子进程
      exit($i);
    }
  }
  //等待子进程执行完毕,避免出现僵尸进程
  while (pcntl_waitpid(0,$status) != -1) {
    $status = pcntl_wexitstatus($status);
    echo "Child $status completedn";
  }

<?php
   for ($i = 0; $i < 10; $i++) {
     $pid = pcntl_fork();
     if ($pid == -1) {
        echo "Could not fork!n";
        exit(1);
     }
     if (!$pid) {
        $redis = PRedis::getInstance();
        // do something   
        exit;
     }
   }

<?php
   public static function getInstance() {
     static $instances = array();
     $key = getmypid();//获取当前进程ID
     if ($empty($instances[$key])) {
        $inctances[$key] = new self();
     }
   
     return $instances[$key];
   }

function microtime_float()
{
   list($u_sec,$sec) = explode(' ',microtime());
   return (floatval($u_sec) + floatval($sec));
}
$start_time = microtime_float();
//do something
usleep(100);
$end_time = microtime_float();
$total_time = $end_time - $start_time;
$time_cost = sprintf("%.10f",$total_time);
echo "program cost total " . $time_cost . "sn";