Commit 15ea30c8 authored by Roque's avatar Roque

Revert "[WIP] http requests are done using curl command"

This reverts commit 469eb486.
http requests issue (see nexedi/ebulk#1) solved
parent 469eb486
...@@ -4,7 +4,6 @@ require 'openssl' ...@@ -4,7 +4,6 @@ require 'openssl'
require 'yaml' require 'yaml'
require 'open-uri' require 'open-uri'
require_relative 'filelogger' require_relative 'filelogger'
require 'open3'
# class representing a Wendelin client # class representing a Wendelin client
class WendelinClient class WendelinClient
...@@ -25,37 +24,6 @@ class WendelinClient ...@@ -25,37 +24,6 @@ class WendelinClient
@banned_references_list = [] @banned_references_list = []
@logger = LogManager.instance() @logger = LogManager.instance()
@last_ingestion = Time.new - 2 @last_ingestion = Time.new - 2
begin
# check credentials
uri = URI(URI.escape("#{@erp5_url}"))
uri = URI(URI.escape("#{uri.scheme}://#{uri.host}/erp5/getDataStreamList?data_set_reference="))
scheme = uri.scheme == 'https' ? "https://" : "http://"
check_credential_url = uri.to_s.sub! scheme, scheme + @user + ":" + @password + "@"
command = "curl --request GET '" + check_credential_url + "'"
stdout,stderr,status = Open3.capture3(command)
@logger.info("Checking credentials...")
print "[INFO] Checking credentials..."
if status.success?
if stdout.include? "'"
stdout = stdout.gsub! "'", '"'
end
dict = JSON.parse(stdout)
if dict["status_code"] == 1
@logger.error(dict["error_message"], print=TRUE)
@logger.abortExecution()
end
@logger.info("OK")
puts "OK"
else
@logger.error(HTTP_MESSAGE_400, print=TRUE)
@logger.error(stderr)
@logger.abortExecution()
end
rescue Exception => e
@logger.error("HTTP ERROR: " + e.to_s, print=TRUE)
@logger.error(e.backtrace)
@logger.abortExecution()
end
end end
def checkReferenceChars(reference) def checkReferenceChars(reference)
...@@ -68,9 +36,7 @@ class WendelinClient ...@@ -68,9 +36,7 @@ class WendelinClient
checkReferenceChars(reference) checkReferenceChars(reference)
uri = URI(URI.escape("#{@erp5_url}/ingestionReferenceExists?reference=#{reference}")) uri = URI(URI.escape("#{@erp5_url}/ingestionReferenceExists?reference=#{reference}"))
begin begin
response = handleRequest(uri, nil, nil, non_json_content=true) res = open(uri, http_basic_authentication: [@user, @password]).read
res = response["message"] if response["success"] == TRUE
raise response["message"] if response["success"] == FALSE
rescue Exception => e rescue Exception => e
@logger.error("An error occurred while checking if reference exists: " + e.to_s) @logger.error("An error occurred while checking if reference exists: " + e.to_s)
@logger.error(e.backtrace) @logger.error(e.backtrace)
...@@ -87,7 +53,7 @@ class WendelinClient ...@@ -87,7 +53,7 @@ class WendelinClient
@logger.info("Deletion requested for reference #{reference}", print=TRUE) @logger.info("Deletion requested for reference #{reference}", print=TRUE)
checkReferenceChars(reference) checkReferenceChars(reference)
uri = URI(URI.escape("#{@erp5_url}/ERP5Site_invalidateIngestionObjects?reference=#{reference}")) uri = URI(URI.escape("#{@erp5_url}/ERP5Site_invalidateIngestionObjects?reference=#{reference}"))
res = handleRequest(uri, nil, nil, non_json_content=true) res = handleRequest(uri)
if res["success"] == FALSE if res["success"] == FALSE
@logger.abortExecution() @logger.abortExecution()
end end
...@@ -99,7 +65,7 @@ class WendelinClient ...@@ -99,7 +65,7 @@ class WendelinClient
checkReferenceChars(reference) checkReferenceChars(reference)
checkReferenceChars(new_reference) checkReferenceChars(new_reference)
uri = URI(URI.escape("#{@erp5_url}/ERP5Site_renameIngestion?reference=#{reference}&new_reference=#{new_reference}")) uri = URI(URI.escape("#{@erp5_url}/ERP5Site_renameIngestion?reference=#{reference}&new_reference=#{new_reference}"))
res = handleRequest(uri, nil, nil, non_json_content=true) res = handleRequest(uri)
if res["success"] == FALSE if res["success"] == FALSE
@logger.abortExecution() @logger.abortExecution()
end end
...@@ -112,9 +78,8 @@ class WendelinClient ...@@ -112,9 +78,8 @@ class WendelinClient
else else
@logger.info("Increasing dataset version") @logger.info("Increasing dataset version")
begin begin
uri = URI(URI.escape("#{@erp5_url}/ERP5Site_increaseDatasetVersion?reference=#{reference}")) uri = URI(URI.escape("#{@erp5_url}/ERP5Site_increaseDatasetVersion?reference=#{reference}"))
response = handleRequest(uri, nil, nil, non_json_content=true) res = open(uri, http_basic_authentication: [@user, @password]).read
raise response["message"] if response["success"] == FALSE
rescue Exception => e rescue Exception => e
@logger.error("An error occurred while increasing dataset version: " + e.to_s) @logger.error("An error occurred while increasing dataset version: " + e.to_s)
@logger.error(e.backtrace) @logger.error(e.backtrace)
...@@ -185,27 +150,31 @@ class WendelinClient ...@@ -185,27 +150,31 @@ class WendelinClient
success = FALSE success = FALSE
n_retry = 0 n_retry = 0
while ! success && n_retry < 10 while ! success && n_retry < 10
response = handleRequest(uri, nil, nil, non_json_content=true) begin
if response["success"] == TRUE res = open(uri, http_basic_authentication: [@user, @password]) {
chunk = response["message"] |content|
if chunk.nil? || chunk.empty? chunk = content.read()
if first if chunk.nil? || chunk.empty?
yield chunk if first
end yield chunk
@logger.info("Done", print=TRUE) end
done = TRUE @logger.info("Done", print=TRUE)
else done = TRUE
first = FALSE else
n_part += 1 first = FALSE
yield chunk n_part += 1
end yield chunk
success = TRUE end
else }
@logger.error("Error downloading data: " + response["message"], print=TRUE) success = TRUE
n_retry += 1 rescue Exception => e
@logger.info("Retrying #{n_retry}/10...", print=TRUE) exception = e
sleep 30 @logger.error("Error downloading data: " + e.to_s, print=TRUE)
end @logger.error(e.backtrace)
n_retry += 1
@logger.info("Retrying #{n_retry}/10...", print=TRUE)
sleep 30
end
end end
raise exception if not success raise exception if not success
end end
...@@ -226,41 +195,55 @@ class WendelinClient ...@@ -226,41 +195,55 @@ class WendelinClient
end end
private private
def handleRequest(uri, reference=nil, data_chunk=nil, non_json_content=false) def handleRequest(uri, reference=nil, data_chunk=nil)
scheme = uri.scheme == 'https' ? "https://" : "http://" req = Net::HTTP::Post.new(uri)
url = uri.to_s.sub! scheme, scheme + @user + ":" + @password + "@" req.basic_auth @user, @password
begin
command = "curl --request POST '" + url + "'"
if data_chunk != nil if data_chunk != nil
command += " --data 'data_chunk=" + data_chunk + "'" @logger.info("Setting request form data...", print=TRUE) if reference != nil
begin
req.set_form_data('data_chunk' => data_chunk)
rescue java.lang.OutOfMemoryError
@logger.logOutOfMemoryError(reference)
return {"success"=>FALSE, "message"=>HTTP_MEMORY_ERROR}
rescue Exception => e
@logger.error("Error setting form data: " + e.to_s, print=TRUE)
@logger.error(e.backtrace)
return {"success"=>FALSE, "message"=>HTTP_MESSAGE_EXCEPTION}
end
@logger.info("Sending record:'#{reference}'...", print=TRUE) if reference != nil
end end
stdout,stderr,status = Open3.capture3(command)#) + " >&2") begin
if status.success? res = Net::HTTP.start(uri.hostname, uri.port,
# check response (if ok, it should be empty or json) :use_ssl => (uri.scheme == 'https'),
if not stdout.nil? and not stdout.empty? :verify_mode => OpenSSL::SSL::VERIFY_NONE,
begin :ssl_timeout => 300, :open_timeout => 300, :read_timeout => 300,
if not non_json_content ) do |http|
if stdout.include? "'" http.request(req)
stdout = stdout.gsub! "'", '"' end
end rescue Exception => e
dict = JSON.parse(stdout) @logger.error("HTTP ERROR: " + e.to_s, print=TRUE)
end @logger.error(e.backtrace)
rescue Exception => e return {"success"=>FALSE, "message"=>HTTP_MESSAGE_EXCEPTION}
@logger.error("EXCEPTION: " + e.to_s, print=TRUE)
@logger.error(HTTP_MESSAGE_400, print=TRUE)
@logger.error(stdout)
end
end
@logger.info("Done")
return {"success"=>TRUE, "message"=>stdout}
else else
@logger.error(HTTP_MESSAGE_OTHER, print=TRUE) if res.kind_of?(Net::HTTPSuccess) # res.code is 2XX
@logger.error(stderr) @logger.info("Done")
end return {"success"=>TRUE, "message"=>res.body}
rescue Exception => e else
@logger.error("HTTP ERROR: " + e.to_s, print=TRUE) @logger.error("HTTP FAIL - code: #{res.code}", print=TRUE)
@logger.error(e.backtrace) if res.code == '500' or res.code == '502' or res.code == '503'
return {"success"=>FALSE, "message"=>e.to_s} @logger.error(HTTP_MESSAGE_5XX, print=TRUE)
end elsif res.code == '401'
@logger.error(HTTP_MESSAGE_401, print=TRUE)
@logger.abortExecution()
elsif res.code == '400'
@logger.error(HTTP_MESSAGE_400, print=TRUE)
@logger.abortExecution()
else
@logger.error(HTTP_MESSAGE_OTHER, print=TRUE)
end
return {"success"=>FALSE, "message"=>HTTP_MESSAGE_NOT_2XX}
end
end
end end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment