Logo Search packages:      
Sourcecode: jets3t version File versions  Download package

FileComparerResults org::jets3t::service::utils::FileComparer::buildDiscrepancyLists ( Map  filesMap,
Map  s3ObjectsMap,
BytesProgressWatcher  progressWatcher 
) throws NoSuchAlgorithmException, FileNotFoundException, IOException, ParseException [inline]

Compares the contents of a directory on the local file system with the contents of an S3 resource. This comparison is performed on a map of files and a map of S3 objects previously generated using other methods in this class.

Parameters:
filesMap a map of keys/Files built using the method buildFileMap(File, String, boolean)
s3ObjectsMap a map of keys/S3Objects built using the method buildS3ObjectMap(S3Service, S3Bucket, String, boolean, S3ServiceEventListener)
progressWatcher watches the progress of file hash generation.
Returns:
an object containing the results of the file comparison.
Exceptions:
NoSuchAlgorithmException 
FileNotFoundException 
IOException 
ParseException 

Definition at line 812 of file FileComparer.java.

References org::jets3t::service::model::BaseS3Object::containsMetadata(), org::jets3t::service::Jets3tProperties::getBoolProperty(), org::jets3t::service::model::S3Object::getContentLength(), org::jets3t::service::model::S3Object::getContentType(), org::jets3t::service::model::S3Object::getKey(), org::jets3t::service::model::S3Object::getLastModifiedDate(), org::jets3t::service::model::S3Object::getMd5HashAsBase64(), org::jets3t::service::model::BaseS3Object::getMetadata(), and org::jets3t::service::Jets3tProperties::getStringProperty().

    {
        Set onlyOnServerKeys = new HashSet();
        Set updatedOnServerKeys = new HashSet();
        Set updatedOnClientKeys = new HashSet();
        Set alreadySynchronisedKeys = new HashSet();
        Set onlyOnClientKeys = new HashSet();

        // Read property settings for file comparison.
        boolean useMd5Files = jets3tProperties
            .getBoolProperty("filecomparer.use-md5-files", false);
        boolean generateMd5Files = jets3tProperties
            .getBoolProperty("filecomparer.generate-md5-files", false);
        boolean assumeLocalLatestInMismatch = jets3tProperties
            .getBoolProperty("filecomparer.assume-local-latest-in-mismatch", false);
        String md5FilesRootDirectoryPath = jets3tProperties
            .getStringProperty("filecomparer.md5-files-root-dir", null);
        File md5FilesRootDirectory = null;
        if (md5FilesRootDirectoryPath != null) {
            md5FilesRootDirectory = new File(md5FilesRootDirectoryPath);
            if (!md5FilesRootDirectory.isDirectory()) {
                  throw new FileNotFoundException(
                        "filecomparer.md5-files-root-dir path is not a directory: "
                        + md5FilesRootDirectoryPath);
            }
        }

        // Check files on server against local client files.
        Iterator s3ObjectsMapIter = s3ObjectsMap.entrySet().iterator();
        while (s3ObjectsMapIter.hasNext()) {
            Map.Entry entry = (Map.Entry) s3ObjectsMapIter.next();
            String keyPath = (String) entry.getKey();
            S3Object s3Object = (S3Object) entry.getValue();

            // A special-case check to identify objects created by Panic's
            // Transmit application that serve as directory placehoders -
            // a similar concept to the placeholders JetS3t uses but sadly
            // these look different.
            if (keyPath.endsWith("/")
                && s3Object.getContentLength() == 0
                && "binary/octet-stream".equals(s3Object.getContentType()))
            {
                boolean ignorePanicDirPlaceholders =
                    jets3tProperties.getBoolProperty(
                        "filecomparer.ignore-panic-dir-placeholders", true);

                if (ignorePanicDirPlaceholders) {
                    if (log.isDebugEnabled()) {
                        log.debug("Ignoring object that looks like a directory " +
                        "placeholder created by Panic's Transmit application: " + keyPath);
                    }
                    continue;
                } else {
                    if (log.isWarnEnabled()) {
                        log.warn("Identified an object that looks like a directory " +
                        "placeholder created by Panic's Transmit application. " +
                        "If this object was indeed created by Transmit, it will not " +
                        "be handled properly unless the JetS3t property " +
                        "\"filecomparer.ignore-panic-dir-placeholders\" is set to " +
                        "true. " + s3Object);
                    }
                }
            }

            // Another special-case check, this time for directory placeholder objects
            // created by the S3 Organizer Firefox extension.
            if (keyPath.endsWith("_$folder$")
                && s3Object.getContentLength() == 0
                && "binary/octet-stream".equals(s3Object.getContentType()))
            {
                boolean ignoreS3FoxDirPlaceholders =
                    jets3tProperties.getBoolProperty(
                        "filecomparer.ignore-s3organizer-dir-placeholders", true);

                if (ignoreS3FoxDirPlaceholders) {
                    if (log.isDebugEnabled()) {
                        log.debug("Ignoring object that looks like a directory " +
                        "placeholder created by the S3 Organizer Firefox add-on: " + keyPath);
                    }
                    continue;
                } else {
                    if (log.isWarnEnabled()) {
                        log.warn("Identified an object that looks like a directory " +
                        "placeholder created by the S3 Organizer Firefox add-on. " +
                        "If this object was indeed created by S3 Organizer, it will not " +
                        "be handled properly unless the JetS3t property " +
                        "\"filecomparer.ignore-s3organizer-dir-placeholders\" is set to " +
                        "true. " + s3Object);
                    }
                }
            }

            // Check whether local file is already on server
            if (filesMap.containsKey(keyPath)) {
                // File has been backed up in the past, is it still up-to-date?
                File file = (File) filesMap.get(keyPath);

                if (file.isDirectory()) {
                    // We don't care about directory date changes, as long as it's present.
                    alreadySynchronisedKeys.add(keyPath);
                } else {
                    // Compare file hashes.
                    byte[] computedHash = null;

                    // Check whether a pre-computed MD5 hash file is available
                    File computedHashFile = (md5FilesRootDirectory != null
                        ? new File(md5FilesRootDirectory, keyPath + ".md5")
                        : new File(file.getPath() + ".md5"));
                    if (useMd5Files
                        && computedHashFile.canRead()
                        && computedHashFile.lastModified() > file.lastModified())
                    {
                        BufferedReader br = null;
                        try {
                            // A pre-computed MD5 hash file is available, try to read this hash value
                            br = new BufferedReader(new FileReader(computedHashFile));
                            computedHash = ServiceUtils.fromHex(br.readLine().split("\\s")[0]);
                        } catch (RuntimeException e) {
                            throw e;
                        } catch (Exception e) {
                            if (log.isWarnEnabled()) {
                              log.warn("Unable to read hash from computed MD5 file", e);
                            }
                        } finally {
                            if (br != null)
                              br.close();
                        }
                    }

                    if (computedHash == null) {
                        // A pre-computed hash file was not available, or could not be read.
                        // Calculate the hash value anew.
                        InputStream hashInputStream = null;
                        if (progressWatcher != null) {
                            hashInputStream = new ProgressMonitoredInputStream( // Report on MD5 hash progress.
                                new FileInputStream(file), progressWatcher);
                        } else {
                            hashInputStream = new FileInputStream(file);
                        }
                        computedHash = ServiceUtils.computeMD5Hash(hashInputStream);
                    }

                    String fileHashAsBase64 = ServiceUtils.toBase64(computedHash);

                    if (generateMd5Files && !file.getName().endsWith(".md5") &&
                        (!computedHashFile.exists()
                        || computedHashFile.lastModified() < file.lastModified()))
                    {
                        // Create parent directory for new hash file if necessary
                        File parentDir = computedHashFile.getParentFile();
                        if (parentDir != null && !parentDir.exists()) {
                              parentDir.mkdirs();
                        }

                        // Create or update a pre-computed MD5 hash file.
                        FileWriter fw = null;
                        try {
                            fw = new FileWriter(computedHashFile);
                            fw.write(ServiceUtils.toHex(computedHash));
                        } catch (Exception e) {
                            if (log.isWarnEnabled()) {
                              log.warn("Unable to write computed MD5 hash to a file", e);
                            }
                        } finally {
                            if (fw != null)
                                fw.close();
                        }
                    }

                    // Get the S3 object's Base64 hash.
                    String objectHash = null;
                    if (s3Object.containsMetadata(S3Object.METADATA_HEADER_ORIGINAL_HASH_MD5)) {
                        // Use the object's *original* hash, as it is an encoded version of a local file.
                        objectHash = (String) s3Object.getMetadata(
                            S3Object.METADATA_HEADER_ORIGINAL_HASH_MD5);
                        if (log.isDebugEnabled()) {
                            log.debug("Object in S3 is encoded, using the object's original hash value for: "
                            + s3Object.getKey());
                        }
                    } else {
                        // The object wasn't altered when uploaded, so use its current hash.
                        objectHash = s3Object.getMd5HashAsBase64();
                    }

                    if (fileHashAsBase64.equals(objectHash)) {
                        // Hashes match so file is already synchronised.
                        alreadySynchronisedKeys.add(keyPath);
                    } else {
                        // File is out-of-synch. Check which version has the latest date.
                        Date s3ObjectLastModified = null;
                        String metadataLocalFileDate = (String) s3Object.getMetadata(
                            Constants.METADATA_JETS3T_LOCAL_FILE_DATE);

                        // Try to retrieve the date using the deprecated metadata name
                        if (metadataLocalFileDate == null) {
                            metadataLocalFileDate = (String) s3Object.getMetadata(
                                Constants.METADATA_JETS3T_LOCAL_FILE_DATE_DEPRECATED);
                        }

                        if (metadataLocalFileDate == null) {
                            // This is risky as local file times and S3 times don't match!
                            if (!assumeLocalLatestInMismatch && log.isWarnEnabled()) {
                              log.warn("Using S3 last modified date as file date. This is not reliable "
                                + "as the time according to S3 can differ from your local system time. "
                                + "Please use the metadata item "
                                + Constants.METADATA_JETS3T_LOCAL_FILE_DATE);
                            }
                            s3ObjectLastModified = s3Object.getLastModifiedDate();
                        } else {
                            s3ObjectLastModified = ServiceUtils
                                .parseIso8601Date(metadataLocalFileDate);
                        }
                        if (s3ObjectLastModified.getTime() > file.lastModified()) {
                            updatedOnServerKeys.add(keyPath);
                        } else if (s3ObjectLastModified.getTime() < file.lastModified()) {
                            updatedOnClientKeys.add(keyPath);
                        } else {
                            // Local file date and S3 object date values match exactly, yet the
                            // local file has a different hash. This shouldn't ever happen, but
                            // sometimes does with Excel files.
                            if (assumeLocalLatestInMismatch) {
                                if (log.isWarnEnabled()) {
                                    log.warn("Backed-up S3Object " + s3Object.getKey()
                                    + " and local file " + file.getName()
                                    + " have the same date but different hash values. "
                                    + "Assuming local file is the latest version.");
                                }
                                updatedOnClientKeys.add(keyPath);
                            } else {
                                throw new IOException("Backed-up S3Object " + s3Object.getKey()
                                    + " and local file " + file.getName()
                                    + " have the same date but different hash values. "
                                    + "This shouldn't happen!");
                            }

                        }
                    }
                }
            } else {
                // File is not in local file system, so it's only on the S3
                // server.
                onlyOnServerKeys.add(keyPath);
            }
        }

        // Any local files not already put into another list only exist locally.
        onlyOnClientKeys.addAll(filesMap.keySet());
        onlyOnClientKeys.removeAll(updatedOnClientKeys);
        onlyOnClientKeys.removeAll(alreadySynchronisedKeys);
        onlyOnClientKeys.removeAll(updatedOnServerKeys);

        return new FileComparerResults(onlyOnServerKeys, updatedOnServerKeys, updatedOnClientKeys,
            onlyOnClientKeys, alreadySynchronisedKeys);
    }


Generated by  Doxygen 1.6.0   Back to index