Programmer's Notes » file http://programmersnotes.info Notes on the web-development and artificial intelligence. Mon, 18 Jul 2011 13:20:46 +0000 en hourly 1 http://wordpress.org/?v=3.1 Working with large files eficiently http://programmersnotes.info/2008/06/16/working-with-large-files-eficiently/ http://programmersnotes.info/2008/06/16/working-with-large-files-eficiently/#comments Mon, 16 Jun 2008 16:59:20 +0000 Konstantin Mirin http://konstantin.takeforce.net/?p=3 Yet Another Related Posts Plugin.]]> Once I had a task to import a number of data from datafeeds (that are in CSV or XML) into MySQL database. See solution in my another post. Here I want to tell about handling large files.

Datafeeds there are 50-200M each, so reading all file at once was not possible due to memory limitations. So we need to split it into portions. Here is the class, that solves the problem of downloading remote file, saving it locally and then reading by portions. Position is saved in $this->Offset:

  1. class WebServiceSource implements GenericSource
  2. {
  3.     private $URI;
  4.     private $TempFile;
  5.     private $URIHandler;
  6.     private $TMPHandler;
  7.     private $Data;
  8.     private $Offset = 0;
  9.     private $EOF;
  10.  
  11.     /**
  12.      * Size to read from remote host at once, buffer
  13.      */
  14.     const BUFFER_LENGTH = 102400;//1 MB
  15.     /**
  16.      * Number of records to read at once, from the local file
  17.      */
  18.     const RECORDS_NUM = 15;
  19.  
  20.     public function __construct($uri = NULL)
  21.     {
  22.         $this->SetURI($uri);
  23.         $this->EOF = false;
  24.     }
  25.  
  26.     private function OpenURI()
  27.     {
  28.         $this->URIHandler = @fopen($this->URI, ‘r’);
  29.         if (!$this->URIHandler) throw new ASException(‘WrongDatafeedURL’, ‘DFImport’);
  30.     }
  31.  
  32.     private function OpenTempFile($mode = ‘w+’)
  33.     {
  34.         $this->TMPHandler = @fopen($this->TempFile, $mode);
  35.         if (!$this->TMPHandler) throw new ASException(‘TempFileNotAccessible’, ‘DFImport’);
  36.     }
  37.  
  38.     private function CloseURI()
  39.     {
  40.         fclose($this->URIHandler);
  41.     }
  42.  
  43.     private function CloseTempFile()
  44.     {
  45.         fclose($this->TMPHandler);
  46.     }
  47.  
  48.     private function IsDownloaded()
  49.     {
  50.         if (!file_exists($this->TempFile)) return false;
  51.         if ((time() – filectime($this->TempFile)) > TMP_EXPIRE) return false;
  52.         return true;
  53.     }
  54.  
  55.     private function Download()
  56.     {
  57.         if ($this->IsDownloaded()) return;
  58. //        set_time_limit(0);
  59.         $this->OpenURI();
  60.         $this->OpenTempFile();
  61.         while (!feof($this->URIHandler))
  62.         {
  63.             $buffer = fread($this->URIHandler, self::BUFFER_LENGTH);
  64.             fwrite($this->TMPHandler, $buffer);
  65.         }
  66.         $this->CloseURI();
  67.         $this->CloseTempFile();
  68.     }
  69.  
  70.     private function GetFragment()
  71.     {
  72.         $this->OpenTempFile(‘r’);
  73.         $RecordsNum = 1;
  74.         $this->Data = ”;
  75.         fseek($this->TMPHandler, $this->Offset);
  76.         while (($row = fgets($this->TMPHandler)) && $RecordsNum <= self::RECORDS_NUM)
  77.         {
  78.             $this->Data .= $row;
  79.             $RecordsNum++;
  80.         }
  81.         //if number of records read is less then maximum, then we came to the
  82.         // end of file
  83.         if ($RecordsNum < self::RECORDS_NUM) $this->EOF = true;
  84.         $this->Offset = ftell($this->TMPHandler);
  85.         $this->CloseTempFile();
  86.         return $this->Data;
  87.     }
  88.  
  89.     public function GetData()
  90.     {
  91.         $this->Download();
  92.         if (!$this->EOF)
  93.             return $this->GetFragment();
  94.         return false;
  95.     }
  96.  
  97.     public function SetURI($uri)
  98.     {
  99.         $this->URI = $uri;
  100.         $this->TempFile = SYSTEM_ROOT.TMP_DIR.md5($uri);
  101.     }
  102. }

And here is an example of how it works:

  1. $source = new WebServiceSource($url);
  2. while ($res = $this->Source->GetData())
  3. {
  4.     $this->Parser->SetRawData($res);
  5.     $this->Parser->Parse();
  6. }

No related posts.

Related posts brought to you by Yet Another Related Posts Plugin.

]]> http://programmersnotes.info/2008/06/16/working-with-large-files-eficiently/feed/ 0