PHP: Syncing Data with Amazon S3

Solution:

1. make a list of existing files and their md5sum values
2. Get all of the s3 Objects
3. Compare md5s and transfer if different

4. remove any 'left over' (--delete-removed) files

   function syncS3($bucket, $prefix, $targetDir)
   {
       if (!is_dir($targetDir)) {
           mkdir($targetDir, 0755, true);
       }
   
       $localFileM5Sums = getMd5ForFilesInDir($targetDir);
       $s3Objects       = $this->s3Client->listObjects(['Bucket' => $bucket, 'Prefix' => $prefix]);
   
       foreach ($s3Objects['Contents'] as $contents) {
           $s3Key = $contents['Key'];
           if (substr($s3Key, -1, 1) == '/') { // is a 'directory'
               continue; // skip
           }
   
           $saveAs = $targetDir . DIRECTORY_SEPARATOR . $s3Key;
           if (isset($localFileM5Sums[$saveAs])) { // if we have local copy
               $localMd5 = $localFileM5Sums[$saveAs];
               unset($localFileM5Sums[$saveAs]); // remove so we don't try to delete it later
               if ($localMd5 === trim($contents['ETag'], '"')) { // and md5sumMatches
                   continue; // don't transfer
               }
           }
   
           // transfer it
           if (!is_dir(dirname($saveAs))) {
               mkdir(dirname($saveAs), 0755, true);
           }
           $this->s3Client->getObject([
               'Bucket' => $bucket,
               'Key' => $s3Key,
               'SaveAs' => $saveAs
           ]);
       }
   
       // remove any missing files
       foreach ($localFileM5Sums as $fileName => $md5sum) {
           unlink($fileName);
       }
   }
   
   
   function getMd5ForFilesInDir($targetDir)
   {
       $iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($targetDir));
       $files    = [];
       foreach ($iterator as $file) {
           if (!$file->isDir())
               $files[$file->getPathname()] = md5_file($file->getPathname());
       }
       return $files;
   }
   Copy code