Logo Search packages:      
Sourcecode: jets3t version File versions  Download package

void org::jets3t::apps::synchronize::Synchronize::uploadLocalDirectoryToS3 ( Map  filesMap,
S3Bucket  bucket,
String  rootObjectPath,
String  aclString,
BytesProgressWatcher  progressWatcher 
) throws Exception [inline]

Copies the contents of a local directory to S3, storing them in the given root path.

A set of comparisons is used to determine exactly how the local files differ from the contents of the S3 location, and files are transferred based on these comparisons and options set by the user.

The following S3 Object properties are set when a file is uploaded:

  • The object's key name
  • Content-Length: The size of the uploaded file. This will be 0 for directories, and will differ from the original file if gzip or encryption options are set.
  • Content-Type: Mimetypes#MIMETYPE_JETS3T_DIRECTORY for directories, otherwise a mimetype determined by Mimetypes#getMimetype unless the gzip option is set, in which case the Content-Type is set to application/x-gzip.

The following jets3t-specific metadata items are also set:

Parameters:
filesMap a map of the local Files with '/'-delimited file paths as keys
bucket the bucket to put the objects in (will be created if necessary)
rootObjectPath the root path where objects are put (will be created if necessary)
aclString the ACL to apply to the uploaded object
progressWatcher a class that reports on the progress of this method
Exceptions:
Exception 

Definition at line 315 of file Synchronize.java.

References org::jets3t::service::utils::FileComparerResults::alreadySynchronisedKeys, org::jets3t::service::utils::FileComparer::buildDiscrepancyLists(), org::jets3t::service::utils::FileComparer::buildS3ObjectMapPartial(), org::jets3t::service::Jets3tProperties::containsKey(), org::jets3t::service::multithread::S3ServiceEventAdaptor::getErrorThrown(), org::jets3t::service::Jets3tProperties::getIntProperty(), org::jets3t::service::Jets3tProperties::getStringProperty(), org::jets3t::service::utils::FileComparerResults::merge(), org::jets3t::service::utils::FileComparerResults::onlyOnClientKeys, org::jets3t::service::utils::FileComparerResults::onlyOnServerKeys, org::jets3t::apps::synchronize::Synchronize::LazyPreparedUploadObject::prepareUploadObject(), printProgressLine(), org::jets3t::service::utils::FileComparerResults::updatedOnClientKeys, org::jets3t::service::utils::FileComparerResults::updatedOnServerKeys, and org::jets3t::service::multithread::S3ServiceEventAdaptor::wasErrorThrown().

Referenced by run().

    {
        FileComparerResults mergedDiscrepancyResults = new FileComparerResults();
        String priorLastKey = null;
        String lastFileKeypathChecked = "";
        long totalObjectsListed = 0;

        EncryptionUtil encryptionUtil = null;
        if (isEncryptionEnabled) {
            String algorithm = properties
                .getStringProperty("crypto.algorithm", "PBEWithMD5AndDES");
            encryptionUtil = new EncryptionUtil(cryptoPassword, algorithm, EncryptionUtil.DEFAULT_VERSION);
        }

        // Repeat upload actions until all objects in bucket have been listed.
        do {
            // List objects in S3. Listing may be complete, or partial.
            printProgressLine("Listing objects in S3"
                + (isBatchMode ? " (Batch mode. Objects listed so far: "
                    + totalObjectsListed + ")" : ""));

            PartialObjectListing partialListing = fileComparer.buildS3ObjectMapPartial(
                s3Service, bucket, rootObjectPath, priorLastKey, !isBatchMode,
                isSkipMetadata, serviceEventAdaptor);
            if (serviceEventAdaptor.wasErrorThrown()) {
                throw new Exception("Unable to build map of S3 Objects",
                    serviceEventAdaptor.getErrorThrown());
            }

            // Retrieve details from listing.
            priorLastKey = partialListing.getPriorLastKey();
            Map s3ObjectsMap = partialListing.getObjectsMap();
            totalObjectsListed += partialListing.getObjectsMap().size();

            ArrayList sortedS3ObjectKeys = new ArrayList(s3ObjectsMap.keySet());
            Collections.sort(sortedS3ObjectKeys);

            // Compare the listed objects with the local sytem.
            printProgressLine("Comparing S3 contents with local system");
            FileComparerResults discrepancyResults = fileComparer.buildDiscrepancyLists(
                filesMap, s3ObjectsMap, progressWatcher);

            // Merge S3 objects and discrepancies to track overall changes.
            mergedDiscrepancyResults.merge(discrepancyResults);

            // Sort upload file candidates by path.
            ArrayList sortedFilesKeys = new ArrayList(filesMap.keySet());
            Collections.sort(sortedFilesKeys);

            List objectsToUpload = new ArrayList();

            // Iterate through local files and perform the necessary action to synchronise them with S3.
            Iterator fileKeyIter = sortedFilesKeys.iterator();
            while (fileKeyIter.hasNext()) {
                String relativeKeyPath = (String) fileKeyIter.next();

                String targetKey = relativeKeyPath;
                if (rootObjectPath.length() > 0) {
                    if (rootObjectPath.endsWith(Constants.FILE_PATH_DELIM)) {
                        targetKey = rootObjectPath + targetKey;
                    } else {
                        targetKey = rootObjectPath + Constants.FILE_PATH_DELIM + targetKey;
                    }
                }

                if (isBatchMode) {
                    if (priorLastKey != null && targetKey.compareTo(priorLastKey) > 0) {
                        // We do not yet have the S3 object listing to compare this file.
                        continue;
                    }

                    if (targetKey.compareTo(lastFileKeypathChecked) <= 0) {
                        // We have already handled this file in a prior batch.
                        continue;
                    } else {
                        lastFileKeypathChecked = targetKey;
                    }
                }

                File file = (File) filesMap.get(relativeKeyPath);

                if (discrepancyResults.onlyOnClientKeys.contains(relativeKeyPath)) {
                    printOutputLine("N " + targetKey, REPORT_LEVEL_ACTIONS);
                    objectsToUpload.add(new LazyPreparedUploadObject(targetKey, file, aclString, encryptionUtil));
                } else if (discrepancyResults.updatedOnClientKeys.contains(relativeKeyPath)) {
                    printOutputLine("U " + targetKey, REPORT_LEVEL_ACTIONS);
                    objectsToUpload.add(new LazyPreparedUploadObject(targetKey, file, aclString, encryptionUtil));
                } else if (discrepancyResults.alreadySynchronisedKeys.contains(relativeKeyPath)) {
                    if (isForce) {
                        printOutputLine("F " + targetKey, REPORT_LEVEL_ACTIONS);
                        objectsToUpload.add(new LazyPreparedUploadObject(targetKey, file, aclString, encryptionUtil));
                    } else {
                        printOutputLine("- " + targetKey, REPORT_LEVEL_ALL);
                    }
                } else if (discrepancyResults.updatedOnServerKeys.contains(relativeKeyPath)) {
                    // This file has been updated on the server-side.
                    if (isKeepFiles) {
                        printOutputLine("r " + targetKey, REPORT_LEVEL_DIFFERENCES);
                    } else {
                        printOutputLine("R " + targetKey, REPORT_LEVEL_ACTIONS);
                        objectsToUpload.add(new LazyPreparedUploadObject(targetKey, file, aclString, encryptionUtil));
                    }
                } else {
                    // Uh oh, program error here. The safest thing to do is abort!
                    throw new SynchronizeException("Invalid discrepancy comparison details for file "
                        + file.getPath()
                        + ". Sorry, this is a program error - aborting to keep your data safe");
                }
            }

            int uploadBatchSize = objectsToUpload.size();
            if ((isEncryptionEnabled || isGzipEnabled)
                && properties.containsKey("upload.transformed-files-batch-size"))
            {
                // Limit uploads to small batches in batch mode -- based on the
                // number of upload threads that are available.
                uploadBatchSize = properties.getIntProperty("upload.transformed-files-batch-size", 1000);
                partialUploadObjectsTotal = objectsToUpload.size();
                partialUploadObjectsProgressCount = 0;
            } else {
                partialUploadObjectsTotal = -1;
            }

            // Upload New/Updated/Forced/Replaced objects to S3.
            while (doAction && objectsToUpload.size() > 0) {
                S3Object[] objects = null;
                if (uploadBatchSize > objectsToUpload.size()) {
                    objects = new S3Object[objectsToUpload.size()];
                } else {
                    objects = new S3Object[uploadBatchSize];
                }

                // Invoke lazy upload object creator.
                for (int i = 0; i < objects.length; i++) {
                    LazyPreparedUploadObject lazyObj =
                        (LazyPreparedUploadObject) objectsToUpload.remove(0);
                    objects[i] = lazyObj.prepareUploadObject();
                }

                (new S3ServiceMulti(s3Service, serviceEventAdaptor)).putObjects(bucket, objects);
                if (serviceEventAdaptor.wasErrorThrown()) {
                    Throwable thrown = serviceEventAdaptor.getErrorThrown();
                    if (thrown instanceof Exception) {
                        throw (Exception) thrown;
                    } else {
                        throw new Exception(thrown);
                    }
                }
                partialUploadObjectsProgressCount += objects.length;
            }
        } while (priorLastKey != null);

        // Delete objects on S3 that don't correspond with local files.
        List objectsToDelete = new ArrayList();
        Iterator serverOnlyIter = mergedDiscrepancyResults.onlyOnServerKeys.iterator();
        while (serverOnlyIter.hasNext()) {
            // Relative key
            String relativeKeyPath = (String) serverOnlyIter.next();

            // Build absolute key path for object.
            String targetKey = relativeKeyPath;
            if (rootObjectPath.length() > 0) {
                if (rootObjectPath.endsWith(Constants.FILE_PATH_DELIM)) {
                    targetKey = rootObjectPath + targetKey;
                } else {
                    targetKey = rootObjectPath + Constants.FILE_PATH_DELIM + targetKey;
                }
            }
            S3Object s3Object = new S3Object(targetKey);

            if (isKeepFiles || isNoDelete) {
                printOutputLine("d " + relativeKeyPath, REPORT_LEVEL_DIFFERENCES);
            } else {
                printOutputLine("D " + relativeKeyPath, REPORT_LEVEL_ACTIONS);
                if (doAction) {
                    objectsToDelete.add(s3Object);
                }
            }
        }
        if (objectsToDelete.size() > 0) {
            S3Object[] objects = (S3Object[]) objectsToDelete.toArray(new S3Object[objectsToDelete.size()]);
            (new S3ServiceMulti(s3Service, serviceEventAdaptor)).deleteObjects(bucket, objects);
            if (serviceEventAdaptor.wasErrorThrown()) {
                Throwable thrown = serviceEventAdaptor.getErrorThrown();
                if (thrown instanceof Exception) {
                    throw (Exception) thrown;
                } else {
                    throw new Exception(thrown);
                }
            }
        }

        // Delete local files that have been moved to S3.
        List filesMoved = new ArrayList();
        if (isMoveEnabled) {
            filesMoved.addAll(mergedDiscrepancyResults.onlyOnClientKeys);
            filesMoved.addAll(mergedDiscrepancyResults.updatedOnClientKeys);
            filesMoved.addAll(mergedDiscrepancyResults.updatedOnServerKeys);
            filesMoved.addAll(mergedDiscrepancyResults.alreadySynchronisedKeys);

            ArrayList dirsToDelete = new ArrayList();
            Iterator filesMovedIter = filesMoved.iterator();
            while (filesMovedIter.hasNext()) {
                String keyPath = (String) filesMovedIter.next();
                File file = (File) filesMap.get(keyPath);

                printOutputLine("M " + keyPath, REPORT_LEVEL_ACTIONS);
                if (doAction) {
                    if (file.isDirectory()) {
                        // Delete directories later, as they may still contain
                        // files until this loop completes.
                        dirsToDelete.add(file);
                    } else {
                        file.delete();
                    }
                }
            }
            Iterator dirIter = dirsToDelete.iterator();
            while (dirIter.hasNext()) {
                File dir = (File) dirIter.next();
                dir.delete();
            }
        }

        printOutputLine(
            doAction ? "" : "[No Action] " +
            "New files: " + mergedDiscrepancyResults.onlyOnClientKeys.size() +
            ", Updated: " + mergedDiscrepancyResults.updatedOnClientKeys.size() +
            (isKeepFiles?
                ", Kept: " +
                (mergedDiscrepancyResults.updatedOnServerKeys.size())
                :
                ", Reverted: " + mergedDiscrepancyResults.updatedOnServerKeys.size()
                ) +
            (isNoDelete || isKeepFiles?
                ", Not Deleted: " + mergedDiscrepancyResults.onlyOnServerKeys.size()
                :
                ", Deleted: " + mergedDiscrepancyResults.onlyOnServerKeys.size()
                ) +
            (isForce ?
                ", Forced updates: " + mergedDiscrepancyResults.alreadySynchronisedKeys.size() :
                ", Unchanged: " + mergedDiscrepancyResults.alreadySynchronisedKeys.size()
                ) +
            (isMoveEnabled ?
                ", Moved: " + filesMoved.size()
                : ""
                ), REPORT_LEVEL_NONE
            );
    }


Generated by  Doxygen 1.6.0   Back to index