require_relative 'filelogger'
require 'digest/md5'
require 'fileutils'

# class that handles dataset tasks report
class DatasetUtils

  DATASET_REPORT_FILE = ".dataset-task-report"
  DATASET_TEMP_REPORT_FILE = ".temp-dataset-task-report"
  DATASET_COMPLETED_FILE = ".dataset-completed"
  RESUME_OPERATION_FILE = ".resume-operation"
  PARTIAL_INGESTION_FILE = ".partial-ingestion"
  STAGED_FILE = ".staged"
  DISCARD_CHANGES_FILE = ".discard-changes"
  SPLIT_FILE = ".split-operation"
  SPLIT_CONTROL_FILE = ".control-split-operation"
  FIRST_INGESTION_FILE = ".first-ingestion"

  RUN_DONE = "done"
  RUN_ERROR = "error"
  RUN_ABORTED = "aborted"
  DELETE = "DELETE"
  RENAME = "RENAME"
  INGESTION = "ingestion"
  DOWNLOAD = "download"
  ADD = "add"
  REMOVE = "remove"
  STATUS_NEW = "new"
  STATUS_RENAMED = "renamed"
  STATUS_MODIFIED = "modified"
  STATUS_DELETED = "deleted"
  STAGE_ADD="add"
  STAGE_REMOVE="remove"
  STAGE_RESET="reset"

  OUTPUT_NEW = "new:         "
  OUTPUT_ADD = "add:         "
  OVERWRITE = "overwrite:   "
  OUTPUT_MODIFIED = "modified:    "
  OUTPUT_DELETED = "deleted:     "
  OUTPUT_RENAMED = "renamed:     "

  MEGA = 1000000
  EOF = "EOF"
  CHUNK_SIZE = 50000000 #50mb
  NONE_EXT = "none"
  REFERENCE_SEPARATOR = "/"
  RECORD_SEPARATOR = ";"
  DATE_FORMAT = "%Y-%m-%d-%H-%M-%S"

  def initialize(data_set_directory)
    @data_set_directory = appendSlashTo(data_set_directory)
    @logger = LogManager.instance()
    @task_report_file = @data_set_directory + DATASET_REPORT_FILE
    @temp_report_file = @data_set_directory + DATASET_TEMP_REPORT_FILE
    @completed_file = @data_set_directory + DATASET_COMPLETED_FILE
    @resume_operation_file = @data_set_directory + RESUME_OPERATION_FILE
    @partial_ingestion_file = @data_set_directory + PARTIAL_INGESTION_FILE
    @first_ingestion_file = @data_set_directory + FIRST_INGESTION_FILE
    @staged_file = @data_set_directory + STAGED_FILE
    @discard_changes_file = @data_set_directory + DISCARD_CHANGES_FILE
    @split_file = @data_set_directory + SPLIT_FILE
    @split_control_file = @data_set_directory + SPLIT_CONTROL_FILE
  end

  def getLocalPaths(paths)
    return paths.map {|path|
		      next [] unless Dir.exist?(path)
		      Dir[(path + '/**/*').gsub! '//', '/']
		     }.flatten.select{ |file| File.file?(file) }
  end

  def getLocalFiles(remove=nil)
    local_files = {}
    begin
      File.readlines(@task_report_file).each do |line|
	record = line.split(RECORD_SEPARATOR)
	if record[1].chomp == RUN_DONE
	  if (remove.nil?) || (remove != record[0])
	    local_files[record[0]] = {"size" => record[2].chomp, "hash" => record[3].chomp,
				      "status" => record[1].chomp, "modification_date" => record[4].chomp }
	  end
	end
      end
    rescue Exception => e
      @logger.error("An error occurred in DatasetUtils method 'getLocalFiles':" + e.to_s)
      @logger.error(e.backtrace)
    end
    return local_files
  end

  def saveReport(local_files)
    begin
      File.delete(@temp_report_file) if File.exist?(@temp_report_file)
      if local_files.empty?
	File.open(@temp_report_file, 'w') {}
      else
	local_files.each do |key, array|
	  record = [key, array["status"], array["size"].to_s, array["hash"], array["modification_date"]].join(RECORD_SEPARATOR)
	  File.open(@temp_report_file, 'ab') { |file| file.puts(record) }
	end
      end
      FileUtils.cp_r(@temp_report_file, @task_report_file, :remove_destination => true)
    rescue Exception => e
      @logger.error("An error occurred in DatasetUtils method 'saveReport':" + e.to_s)
      @logger.error(e.backtrace)
    end
  end

  def removeCurrentOperation()
    if File.exist?(@resume_operation_file)
      File.delete(@resume_operation_file)
    end
  end

  def saveCurrentOperation(operation, reference, new_reference)
    if File.exist?(@resume_operation_file)
      File.delete(@resume_operation_file)
    end
    record = new_reference ? [operation, reference, new_reference].join(RECORD_SEPARATOR) : [operation, reference].join(RECORD_SEPARATOR)
    File.open(@resume_operation_file, 'w') { |file| file.puts(record) }
  end

  def reportUpToDate(data_stream_dict, data_set)
    begin
      # directory never downloaded -new or used for partial ingestions-
      return TRUE if not reportFileExist() and not completedFileExist()
      # download not finished
      return FALSE if reportFileExist() and not completedFileExist()
      return FALSE if data_stream_dict["status_code"] == 2
      return TRUE if data_stream_dict["status_code"] != 0
      return TRUE if data_stream_dict["result"].empty?
      changes = getRemoteChangedDataStreams(data_stream_dict["result"], data_set)
      if changes.empty?
	return TRUE
      elsif changes.length == 1
	# check if the unique detected change corresponds to an interrupted ingestion
	if File.exist?(@resume_operation_file)
	  operation=File.open(@resume_operation_file).read.chomp.split(RECORD_SEPARATOR)
	  if operation[1] == changes[0]["reference"]
	    @logger.info("File '#{operation[1]}' was detected as a change in remote dataset, but it is a false positive.")
	    @logger.info("That file operation ingestion was interrupted right after successfully ingest it but local report is outdated.")
	    @logger.info("Dataset is up to date, file operation was reported as ingested in local report.", print=TRUE)
	    new_reference = operation[0] == RENAME ? changes[0]["new_reference"] : FALSE
	    if new_reference
	      file_path = referenceToPath(new_reference, @data_set_directory, data_set)
	    else
	      file_path = referenceToPath(changes[0]["reference"], @data_set_directory, data_set)
	    end
	    size = changes[0]["size"]
	    hash = changes[0]["hash"]
	    addToReport(changes[0]["reference"], RUN_DONE, size, hash, data_set, new_reference)
	    File.delete(@resume_operation_file)
	    return TRUE
	  end
	end
      end
      return FALSE
    rescue Exception => e
      @logger.error("An error occurred in DatasetUtils method 'reportUpToDate':" + e.to_s)
      @logger.error(e.backtrace)
      return FALSE
    end
  end

  def showChanges(changes, status)
    changes.each do |change|
      if status != ""
	status_output = status
      elsif change["status"] == STATUS_NEW
	status_output = OUTPUT_NEW
      elsif change["status"] == STATUS_MODIFIED
	status_output = OUTPUT_MODIFIED
      elsif change["status"] == STATUS_DELETED
	status_output = OUTPUT_DELETED
      elsif change["status"] == STATUS_RENAMED
	status_output = OUTPUT_RENAMED
      else
	status_output = "change:   "
      end
      path = status != OVERWRITE ? change["path"] : change
      new_path = change["status"] == STATUS_RENAMED ? " --> #{change["new_path"]}" : ""
      @logger.info("   #{status_output}#{path}#{new_path}", print=TRUE)
    end
  end

  def showChangesList(changes, message, print_short, status="")
    if not changes.empty?
      if message and message != ""
	@logger.info(message, print=TRUE)
      end
      if print_short and changes.length > 200
	limit = changes.length > 300 ? 100 : changes.length/3
	showChanges(changes[0, limit], status)
	puts "....."
	showChanges(changes[changes.length-limit, changes.length-1], status)
      else
	showChanges(changes, status)
      end
    end
  end

  def showTaskReport(task_reports)
    return if task_reports.empty?
    @logger.info("Reports:", print=TRUE)
    if task_reports.length > 15
      @logger.info(task_reports[0, 5], print=TRUE)
      puts "....."
      @logger.info(task_reports[task_reports.length-5, task_reports.length-1], print=TRUE)
      @logger.info("Full task report:")
      @logger.info(task_reports)
    else
      @logger.info(task_reports, print=TRUE)
    end
  end

  def showTaskErrors(failed_tasks, storage=FALSE)
    puts
    @logger.error("The following files could not be processed. Please check the details in the log file: " + @logger.getLogPath(), print=TRUE)
    if failed_tasks.length > 15
      @logger.error(failed_tasks[0, 5], print=TRUE)
      puts "....."
      @logger.error(failed_tasks[failed_tasks.length-5, failed_tasks.length-1], print=TRUE)
    else
      @logger.error(failed_tasks, print=TRUE)
    end
    if storage
      puts
      @logger.info("You can retry the operation for those files by running your command again with parameter --restart .", print=TRUE)
    else
      @logger.info("You can retry the operation for those files.", print=TRUE)
    end
    puts
  end

  def splitOperationControlFileExist(reference)
    file_reference = reference.gsub('/', '__')
    return File.exist?(@split_control_file + "__" + file_reference)
  end

  def createSplitOperationControlFile(reference)
    file_reference = reference.gsub('/', '__')
    File.open(@split_control_file + "__" + file_reference, 'w') {}
  end

  def deleteSplitOperationControlFile(reference=nil)
    if reference.nil?
      Dir.glob("#{@split_control_file}*").each { |file| File.delete(file) }
    else
      file_reference = reference.gsub('/', '__')
      File.delete(@split_control_file + "__" + file_reference) if File.exist?(@split_control_file + "__" + file_reference)
    end
  end

  def deleteSplitOperationFile(reference=nil)
    if reference.nil?
      Dir.glob("#{@split_file}*").each { |file| File.delete(file) }
    else
      file_reference = reference.gsub('/', '__')
      File.delete(@split_file + "__" + file_reference) if File.exist?(@split_file + "__" + file_reference)
    end
  end

  def saveSplitOperation(operation, reference, eof, hash, chunk_size)
    file_reference = reference.gsub('/', '__')
    record = [operation, reference, eof, hash, Integer(chunk_size)].join(RECORD_SEPARATOR)
    File.open(@split_file + "__" + file_reference, 'w') { |file| file.puts(record) }
    deleteSplitOperationControlFile(reference)
  end

  def splitOperationFileExist(reference)
    file_reference = reference.gsub('/', '__')
    return File.exist?(@split_file + "__" + file_reference)
  end

  def getLastSplitOperation(operation, reference, hash, chunk_size)
    file_reference = reference.gsub('/', '__')
    return 0 if not File.exist?(@split_file + "__" + file_reference)
    record = File.open(@split_file + "__" + file_reference).read.chomp.split(RECORD_SEPARATOR)
    if record[0] == operation && record[1] == reference && record[3] == hash && record[4] == Integer(chunk_size).to_s && record[2] != EOF
      # discard if user interrupted (ctrl+c) the operation
      if splitOperationControlFileExist(file_reference)
	@logger.warn("Previous split operation attempt for file #{reference} was interrupt by user (aborted tool execution), it will be restarted.", print=TRUE)
	deleteSplitOperationFile(file_reference)
	deleteSplitOperationControlFile(file_reference)
	return 0
      end
      createSplitOperationControlFile(file_reference)
      return record[2].to_i
    end
    return 0
  rescue Exception => e
    @logger.error("An error occurred in getLastSplitOperation method:" + e.to_s)
    @logger.error(e.backtrace)
    return 0
  end

  def deleteDiscardChangesFile()
    File.delete(@discard_changes_file) if File.exist?(@discard_changes_file)
  end

  def discardChangesFileExist()
    return File.exist?(@discard_changes_file)
  end

  def deleteCompletedFile()
    File.delete(@completed_file) if File.exist?(@completed_file)
  end

  def createCompletedFile()
    File.open(@completed_file, 'w') {}
  end

  def completedFileExist()
    return File.exist?(@completed_file)
  end

  def createReportFile()
    File.open(@task_report_file, 'w') {}
  end

  def reportFileExist()
    return TRUE if File.exist?(@task_report_file)
    if File.exist?(@temp_report_file)
      FileUtils.cp_r(@temp_report_file, @task_report_file)
      return TRUE
    else
      return FALSE
    end
  end

  def deleteFirstIngestionFile()
    File.delete(@first_ingestion_file) if File.exist?(@first_ingestion_file)
  end

  def createFirstIngestionFile()
    File.open(@first_ingestion_file, 'w') {}
  end

  def firstIngestionFileExist()
    return File.exist?(@first_ingestion_file)
  end

  def deletePartialIngestionFile()
    File.delete(@partial_ingestion_file) if File.exist?(@partial_ingestion_file)
  end

  def createPartialIngestionFile()
    File.open(@partial_ingestion_file, 'w') {}
  end

  def partialIngestionFileExist()
    return File.exist?(@partial_ingestion_file)
  end

  def stagedFileExist()
    File.exist?(@staged_file) or File.zero?(@staged_file)
  end

  def appendSlashTo(element)
    return element.end_with?("/") ? element : element + "/"
  end

  def referenceToPath(reference, data_set_directory, dataset)
    data_set = appendSlashTo(dataset)
    file_path = data_set_directory + reference.reverse.sub(REFERENCE_SEPARATOR.reverse, ".".reverse).reverse.sub(data_set, "")
    file_path = file_path[0...-5] if file_path.end_with?(".#{NONE_EXT}")
    return file_path
  end

  def getPathInfo(path, dataset)
    extension = File.extname path
    if path.start_with?(@data_set_directory)
      filename = path.sub(@data_set_directory, "")
      filename = filename.reverse.sub(extension.reverse, "").reverse
    end
    extension.gsub! '.', ''
    extension = extension == "" ? NONE_EXT : extension
    reference = path.sub(@data_set_directory, appendSlashTo(dataset))
    if extension != NONE_EXT
      old_pattern = filename + '.' + extension
      new_pattern = filename + REFERENCE_SEPARATOR + extension
      reference = reference.reverse.sub(old_pattern.reverse, new_pattern.reverse).reverse
    else
      reference += REFERENCE_SEPARATOR + extension
    end
    return filename, extension, reference
  end

  def addToReport(reference, status, size, hash, data_set, new_reference=FALSE)
    local_files = {}
    begin
      file_path = referenceToPath(reference, @data_set_directory, data_set)
      modification_date = File.exist?(file_path) ? File.mtime(file_path).strftime(DATE_FORMAT) : "not-modification-date"
      if not reportFileExist()
	File.open(@task_report_file, 'w') {}
      end
      new_file = TRUE
      File.readlines(@task_report_file).each do |line|
	record = line.split(RECORD_SEPARATOR)
	if reference.to_s == record[0].to_s
	  if new_reference
	    reference = new_reference
	    file_path = referenceToPath(reference, @data_set_directory, data_set)
	    modification_date = File.exist?(file_path) ? File.mtime(file_path).strftime(DATE_FORMAT) : "not-modification-date"
	  end
	  local_files[reference] = {"size" => size, "hash" => hash, "status" => status,
				    "modification_date" => modification_date }
	  new_file = FALSE
	else
	  local_files[record[0]] = {"size" => record[2].chomp, "hash" => record[3].chomp,
				    "status" => record[1].chomp, "modification_date" => record[4].chomp }
	end
      end
      if new_file
	local_files[reference] = {"size" => size, "hash" => hash, "status" => status,
				  "modification_date" => modification_date }
      end
    rescue Exception => e
      @logger.error("An error occurred in DatasetUtils method 'addToReport':" + e.to_s)
      @logger.error(e.backtrace)
    end
    saveReport(local_files)
  end

  def deleteFromReport(reference, status)
    local_files = getLocalFiles(remove=reference)
    saveReport(local_files)
  end

  def getHash(file)
    return "FILE-NOT-EXISTS" if ! File.exist?(file)
    begin
      chunk_size = 4 * MEGA
      md5 = Digest::MD5.new
      open(file) do |f|
	while chunk=f.read(chunk_size)
	  md5.update(chunk)
	end
      end
      return md5.hexdigest
    rescue Exception => e
      @logger.error("An error occurred while getting hash of file " + file + ":" + e.to_s, print=TRUE)
      @logger.error(e.backtrace)
      raise e
    end
  end

  def getLocalConflicts(remote_streams, data_set)
    paths = [appendSlashTo(@data_set_directory)]
    local_files = getLocalPaths(paths)
    ignore, local_changes = getLocalChanges(local_files, data_set, staged=FALSE)
    remote_changes = remote_streams.map { |remote|
	remote = referenceToPath(remote["reference"], @data_set_directory, data_set)
    }
    return local_changes.select{ |conflict| remote_changes.include? conflict["path"] }.map{ |conflict| conflict["path"] }
  end

  def getRemoteConflicts(data_streams, tasks, dataset)
    conflicts = []
    files = tasks.map{ |task| task["path"] }
    data_streams.each do |data_stream|
      file_path = referenceToPath(data_stream["reference"], @data_set_directory, dataset)
      if files.include? file_path
	conflicts.push(file_path.sub(@data_set_directory, "./"))
      end
    end
    return conflicts
  end

  def deleteStagedFile()
    File.delete(@staged_file) if File.exist?(@staged_file)
  end

  def getStagedRecords()
    # build a dictionary due to staged file could contain more than one operation per file
    staged_dict = {}
    if stagedFileExist()
      File.readlines(@staged_file).each_with_index do |line, index|
	record = line.split(RECORD_SEPARATOR)
	path = @data_set_directory + record[1].chomp
	operation = record[0].chomp
	staged_dict[appendSlashTo(path)] = {"index" => index, "operation" => operation}
      end
    end
    return staged_dict
  end

  def updateStagedDeletions(untrucked_deletions)
    untrucked_deletions.each do |path|
      path = path.sub(@data_set_directory, "")
      File.open(@staged_file, 'ab') { |file| file.puts(STAGE_RESET+RECORD_SEPARATOR+path) }
    end
  end

  def isRenamed(file, file_dict_list, file_dict=FALSE)
    hash = file_dict ? file_dict["hash"] : ""
    size = file_dict ? file_dict["size"] : (File.size(file).to_s if File.exist?(file))
    file_dict_list.each do |path, dict|
      if size == dict["size"].to_s
	hash = hash != "" ? hash : getHash(file).to_s
	if hash == dict["hash"]
	  old_path = path
	  return {"key" => old_path, "size" => size, "hash" => hash}
	end
      end
    end
    return FALSE
  end

  def isStaged(path, staged_dict, status)
    return FALSE if staged_dict.nil? || staged_dict.empty?
    staged_status = {"index" => -1, "status" => ""}
    original_file_path = path
    while appendSlashTo(path) != @data_set_directory
      key = appendSlashTo(path) if staged_dict.key?(appendSlashTo(path))
      if ! key.nil?
	if staged_dict[key]["index"] > staged_status["index"]
	  # ignore files marked as added that not exists in filesystem
	  if not (staged_dict[key]["operation"] == STAGE_ADD and not File.exist?(original_file_path))
	    staged_status = {"index" => staged_dict[key]["index"], "status" => staged_dict[key]["operation"]}
	  end
	end
      end
      path = appendSlashTo(path).split("/")[0...-1].join("/")
    end
    return FALSE if staged_status["index"] == -1
    return staged_status["status"] == status
  end

  def checkRenamed(path, deleted_files, change)
    renamed_dict = isRenamed(path, deleted_files)
    if renamed_dict
      deleted_files.delete(renamed_dict["key"])
      change["status"] = STATUS_RENAMED
      change["new_path"] = change["path"]
      change["path"] = renamed_dict["key"]
      change["size"] = renamed_dict["size"]
      change["hash"] = renamed_dict["hash"]
    end
    return change
  end

  def getLocalChanges(files, data_set, staged, partial_ingestion=FALSE)
    staged_changes, untracked_changes = [], []
    staged_dict = getStagedRecords() if staged
    deleted_files = {}
    begin
      if reportFileExist()
	File.readlines(@task_report_file).each do |line|
	  record = line.split(RECORD_SEPARATOR)
	  if record[1].chomp == RUN_DONE
	    file_path = referenceToPath(record[0], @data_set_directory, data_set)
	    if files.include? file_path
	      modification_date = File.mtime(file_path).strftime(DATE_FORMAT)
	      if staged && isStaged(file_path, staged_dict, STAGE_REMOVE)
		staged_changes.push({"path" => file_path, "size" => "", "hash" => DELETE, "status" => STATUS_DELETED })
	      elsif modification_date != record[4].chomp
		size = File.size(file_path).to_s
		hash = getHash(file_path).to_s
		change = {"path" => file_path, "size" => size, "hash" => hash, "status" => STATUS_MODIFIED }
		if size == record[2].to_s
		  if hash != record[3].chomp
		    staged && isStaged(file_path, staged_dict, STAGE_ADD) ? staged_changes.push(change) : untracked_changes.push(change)
		  end
		else
		  staged && isStaged(file_path, staged_dict, STAGE_ADD) ? staged_changes.push(change) : untracked_changes.push(change)
		end
	      end
	      files.delete(file_path)
	    else
	      if not partial_ingestion
		deleted_files[file_path] = {"size" => record[2].to_s, "hash" => record[3].chomp}
	      end
	    end
	  end
	end
      end
      untrucked_deletions = []
      files.each do |path|
	change = {"path" => path, "size" => "", "hash" => "", "status" => STATUS_NEW }
	if staged
	  # if an untracked new file was staged as deletion, delete it
	  if isStaged(path, staged_dict, STAGE_REMOVE)
	    if File.exist?(path)
	      # check scenario where new files were created inside a directory staged as deleted
	      if File.mtime(path) > File.mtime(@staged_file)
		change = checkRenamed(path, deleted_files, change)
		untracked_changes.push(change)
	      else
		File.delete(path)
		@logger.info("Local file added for remove deleted: " + path, print=TRUE)
	      end
	    end
	    untrucked_deletions.push(path)
	  else
	    change = checkRenamed(path, deleted_files, change)
	    isStaged(path, staged_dict, STAGE_ADD) ? staged_changes.push(change) : untracked_changes.push(change)
	  end
	else
	  change = checkRenamed(path, deleted_files, change)
	  untracked_changes.push(change)
	end
      end
      deleted_files.each do |deleted_path, dict|
	change = {"path" => deleted_path, "size" => "", "hash" => DELETE, "status" => STATUS_DELETED }
	staged && isStaged(deleted_path, staged_dict, STAGE_REMOVE) ? staged_changes.push(change) : untracked_changes.push(change)
      end
      updateStagedDeletions(untrucked_deletions) if staged
    rescue Exception => e
      @logger.error("An error occurred in DatasetUtils method 'getLocalChanges':" + e.to_s)
      @logger.error(e.backtrace)
    end
    return staged_changes, untracked_changes
  end

  def dirEmpty(dir)
    Dir[(dir + '/**/*').gsub! '//', '/'].each do |path|
      return FALSE if File.file?(path)
    end
    return TRUE
  end

  def getRemoteFileListForDiscardLocalChanges(data_streams, data_set, check_changes=FALSE, changes=[])
    remotes_for_discard_local = []
    if changes.empty?
      paths = [appendSlashTo(@data_set_directory)]
      local_files = getLocalPaths(paths)
      ignore, local_changes = getLocalChanges(local_files, data_set, staged=FALSE)
    else
      local_changes = changes
    end
    return local_changes if check_changes
    local_changes.each do |change|
      f, e, reference = getPathInfo(change["path"], data_set)
      case change["status"]
	when STATUS_NEW
	  @logger.info("Discarding local change on '#{change["path"]}'", print=TRUE)
	  @logger.info("New file removed.", print=TRUE)
	  puts
	  File.delete(change["path"]) if File.exist?(change["path"])
	when STATUS_RENAMED
	  @logger.info("Discarding local change on '#{change["new_path"]}'", print=TRUE)
	  @logger.info("File moved/renamed back to '#{change["path"]}'.", print=TRUE)
	  puts
	  new_reference = reference
	  f, e, reference = getPathInfo(change["new_path"], data_set)
	  unless File.directory?(File.dirname(change["path"]))
	    FileUtils.mkdir_p(File.dirname(change["path"]))
	  end
	  FileUtils.mv(change["new_path"], change["path"]) if File.exist?(change["new_path"])
	when STATUS_MODIFIED
	  data_stream = data_streams.find{|data_stream| data_stream["reference"] == reference }
	  if data_stream
	    data_stream["path"] = change["path"]
	    remotes_for_discard_local.push(data_stream)
	  end
	when STATUS_DELETED
	  data_stream = data_streams.find{|data_stream| data_stream["reference"] == reference }
	  if data_stream
	    data_stream["path"] = change["path"]
	    remotes_for_discard_local.push(data_stream)
	  end
      end
    end
    return remotes_for_discard_local
  end

  def getRemoteChangedDataStreams(data_streams, data_set)
    changed_data_streams = []
    new_changed_files = {}
    begin
      if reportFileExist()
	local_files = {}
	remote_files = []
	File.readlines(@task_report_file).each do |line|
	  record = line.split(RECORD_SEPARATOR)
	  if record[1].chomp == RUN_DONE
	    local_files[record[0]] = {"size" => record[2].chomp, "hash" => record[3].chomp, }
	  end
	end
	data_streams.each do |data_stream|
	  remote_files.push(data_stream["reference"])
	  pending = TRUE
	  reference = data_stream["reference"]
	  if local_files.has_key? reference
	    size = local_files[reference]["size"]
	    if size.to_s == data_stream["size"].to_s
	      hash = local_files[reference]["hash"]
	      if hash == data_stream["hash"] or data_stream["hash"] == ""
		pending = FALSE
	      end
	    end
	  end
	  if pending
	    local_files.delete(reference)
	    file_path = referenceToPath(data_stream["reference"], @data_set_directory, data_set)
	    new_changed_files[file_path] = data_stream
	  end
	end
	local_files.each do |reference, file_dict|
	  if not remote_files.include? reference
	    changed_data_stream = {"reference" => reference, "hash" => DELETE }
	    file_path = referenceToPath(reference, @data_set_directory, data_set)
	    renamed_dict = isRenamed(file_path, new_changed_files, file_dict)
	    if renamed_dict
	      changed_data_stream = {"reference" => reference, "id" => new_changed_files[renamed_dict["key"]]["id"],
				     "new_reference" => new_changed_files[renamed_dict["key"]]["reference"], "status" => STATUS_RENAMED, 
				     "size" => renamed_dict["size"], "hash" => renamed_dict["hash"] }
	      new_changed_files.delete(renamed_dict["key"])
	    end
	    changed_data_streams.push(changed_data_stream)
	  end
	end
	new_changed_files.each do |path, data_stream|
	  changed_data_streams.push(data_stream)
	end
      end
    rescue Exception => e
      @logger.error("An error occurred in DatasetUtils method 'getRemoteChangedDataStreams':" + e.to_s)
      @logger.error(e.backtrace)
    end
    return changed_data_streams
  end
end
