Skip to content

M
MariaDB
Commit e85006e6 authored by Olivier Bertrand's avatar Olivier Bertrand
Browse files
Options

- Fix bug making REST table fail using CURL 

  This when the HTTP contains & characters
  modified:   storage/connect/tabbson.cpp
  modified:   storage/connect/tabjson.cpp

- Make stringfy option work on only one Json item
  modified:   storage/connect/tabbson.cpp
  modified:   storage/connect/tabbson.h
  modified:   storage/connect/tabjson.cpp
  modified:   storage/connect/tabjson.h

- Make Json/Bson DATE columns accept JSON date syntax
  modified:   storage/connect/tabbson.cpp
  modified:   storage/connect/tabjson.cpp

- Fix bug making REST table default file not being
  erased when dropping the table
  modified:   storage/connect/tabbson.cpp
  modified:   storage/connect/tabjson.cpp
  modified:   storage/connect/tabrest.cpp
  modified:   storage/connect/tabxml.cpp

- Suppress CHAR(36) --> VARCHAR(36) when DEVELOPMENT
  This was fixed in MyClient
  modified:   storage/connect/ha_connect.cc
parent 801a6d50
Hide whitespace changes
Inline Side-by-side
Showing with 87 additions and 61 deletions
storage/connect/ha_connect.cc
View file @ e85006e6
......@@ -5391,12 +5391,7 @@ static bool add_field(String* sql, TABTYPE ttp, const char* field_name, int typ,
int len, int dec, char* key, uint tm, const char* rem,
char* dft, char* xtra, char* fmt, int flag, bool dbf, char v)
{
#if defined(DEVELOPMENT)
// Some client programs regard CHAR(36) as GUID
char var = (len > 255 || len == 36) ? 'V' : v;
#else
char var = (len > 255) ? 'V' : v;
#endif
bool q, error = false;
const char* type = PLGtoMYSQLtype(typ, dbf, var);
......
storage/connect/tabbson.cpp
View file @ e85006e6
/************* tabbson C++ Program Source Code File (.CPP) *************/
/* PROGRAM NAME: tabbson Version 1.0 */
/* (C) Copyright to the author Olivier BERTRAND 2020 */
/* PROGRAM NAME: tabbson Version 1.1 */
/* (C) Copyright to the author Olivier BERTRAND 2020 - 2021 */
/* This program are the BSON class DB execution routines. */
/***********************************************************************/
......@@ -158,8 +158,9 @@ BSONDISC::BSONDISC(PGLOBAL g, uint* lg)
bp = NULL;
row = NULL;
sep = NULL;
strfy = NULL;
i = n = bf = ncol = lvl = sz = limit = 0;
all = strfy = false;
all = false;
} // end of BSONDISC constructor
int BSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
......@@ -173,7 +174,7 @@ int BSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
sep = GetStringTableOption(g, topt, "Separator", ".");
sz = GetIntegerTableOption(g, topt, "Jsize", 1024);
limit = GetIntegerTableOption(g, topt, "Limit", 10);
strfy = GetBooleanTableOption(g, topt, "Stringify", false);
strfy = GetStringTableOption(g, topt, "Stringify", NULL);
/*********************************************************************/
/* Open the input file. */
......@@ -186,6 +187,9 @@ int BSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
#endif // ZIP_SUPPORT
tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL);
if (!tdp->Fn && topt->http)
tdp->Fn = GetStringTableOption(g, topt, "Subtype", NULL);
if (!(tdp->Database = SetPath(g, db)))
return 0;
......@@ -199,7 +203,8 @@ int BSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
if (!tdp->Fn && !tdp->Uri) {
strcpy(g->Message, MSG(MISSING_FNAME));
return 0;
} // endif Fn
} else
topt->subtype = NULL;
if (tdp->Fn) {
// We used the file name relative to recorded datapath
......@@ -428,7 +433,7 @@ bool BSONDISC::Find(PGLOBAL g, PBVAL jvp, PCSZ key, int j)
jcol.Type = TYPE_UNKNOWN;
jcol.Len = jcol.Scale = 0;
jcol.Cbn = true;
} else if (j < lvl) {
} else if (j < lvl && !(strfy && !stricmp(strfy, colname))) {
if (!fmt[bf])
strcat(fmt, colname);
......@@ -499,7 +504,7 @@ bool BSONDISC::Find(PGLOBAL g, PBVAL jvp, PCSZ key, int j)
} // endswitch Type
} else if (lvl >= 0) {
if (strfy) {
if (strfy && !stricmp(strfy, colname)) {
if (!fmt[bf])
strcat(fmt, colname);
......@@ -731,7 +736,6 @@ void BCUTIL::SetJsonValue(PGLOBAL g, PVAL vp, PBVAL jvp)
case TYPE_FLOAT:
switch (vp->GetType()) {
case TYPE_STRING:
case TYPE_DATE:
case TYPE_DECIM:
vp->SetValue_psz(GetString(jvp));
break;
......@@ -749,6 +753,16 @@ void BCUTIL::SetJsonValue(PGLOBAL g, PVAL vp, PBVAL jvp)
if (jvp->Type == TYPE_DBL || jvp->Type == TYPE_FLOAT)
vp->SetPrec(jvp->Nd);
break;
case TYPE_DATE:
if (jvp->Type == TYPE_STRG) {
if (!((DTVAL*)vp)->IsFormatted())
((DTVAL*)vp)->SetFormat(g, "YYYY-MM-DDThh:mm:ssZ", 20, 0);
vp->SetValue_psz(GetString(jvp));
} else
vp->SetValue(GetInteger(jvp));
break;
default:
sprintf(G->Message, "Unsupported column type %d", vp->GetType());
......
storage/connect/tabbson.h
View file @ e85006e6
......@@ -44,10 +44,11 @@ class BSONDISC : public BLOCK {
PBPR row;
PBTUT bp;
PCSZ sep;
PCSZ strfy;
char colname[65], fmt[129], buf[16];
uint *length;
int i, n, bf, ncol, lvl, sz, limit;
bool all, strfy;
bool all;
}; // end of BSONDISC
/***********************************************************************/
......
storage/connect/tabjson.cpp
View file @ e85006e6
/************* tabjson C++ Program Source Code File (.CPP) *************/
/* PROGRAM NAME: tabjson Version 1.8 */
/* (C) Copyright to the author Olivier BERTRAND 2014 - 2020 */
/* (C) Copyright to the author Olivier BERTRAND 2014 - 2021 */
/* This program are the JSON class DB execution routines. */
/***********************************************************************/
#undef BSON_SUPPORT
......@@ -160,8 +160,9 @@ JSONDISC::JSONDISC(PGLOBAL g, uint *lg)
jsp = NULL;
row = NULL;
sep = NULL;
strfy = NULL;
i = n = bf = ncol = lvl = sz = limit = 0;
all = strfy = false;
all = false;
} // end of JSONDISC constructor
int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
......@@ -173,9 +174,9 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth());
lvl = GetIntegerTableOption(g, topt, "Depth", lvl);
sep = GetStringTableOption(g, topt, "Separator", ".");
sz = GetIntegerTableOption(g, topt, "Jsize", 1024);
strfy = GetStringTableOption(g, topt, "Stringify", NULL);
sz = GetIntegerTableOption(g, topt, "Jsize", 250);
limit = GetIntegerTableOption(g, topt, "Limit", 10);
strfy = GetBooleanTableOption(g, topt, "Stringify", false);
/*********************************************************************/
/* Open the input file. */
......@@ -187,6 +188,9 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
#endif // ZIP_SUPPORT
tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL);
if (!tdp->Fn && topt->http)
tdp->Fn = GetStringTableOption(g, topt, "Subtype", NULL);
if (!(tdp->Database = SetPath(g, db)))
return 0;
......@@ -200,7 +204,8 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
if (!tdp->Fn && !tdp->Uri) {
strcpy(g->Message, MSG(MISSING_FNAME));
return 0;
} // endif Fn
} else
topt->subtype = NULL;
if (tdp->Fn) {
// We used the file name relative to recorded datapath
......@@ -426,7 +431,7 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j)
jcol.Type = TYPE_UNKNOWN;
jcol.Len = jcol.Scale = 0;
jcol.Cbn = true;
} else if (j < lvl) {
} else if (j < lvl && !(strfy && !stricmp(strfy, colname))) {
if (!fmt[bf])
strcat(fmt, colname);
......@@ -480,9 +485,8 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j)
strncat(strncat(colname, "_", n), buf, n - 1);
} // endif all
} else {
} else
strncat(fmt, (tdp->Uri ? sep : "[*]"), n);
}
if (Find(g, jar->GetArrayValue(k), "", j))
return true;
......@@ -497,7 +501,7 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j)
} // endswitch Type
} else if (lvl >= 0) {
if (strfy) {
if (strfy && !stricmp(strfy, colname)) {
if (!fmt[bf])
strcat(fmt, colname);
......@@ -1710,7 +1714,6 @@ void JSONCOL::SetJsonValue(PGLOBAL g, PVAL vp, PJVAL jvp)
case TYPE_DTM:
switch (vp->GetType()) {
case TYPE_STRING:
case TYPE_DATE:
vp->SetValue_psz(jvp->GetString(g));
break;
case TYPE_INT:
......@@ -1728,7 +1731,17 @@ void JSONCOL::SetJsonValue(PGLOBAL g, PVAL vp, PJVAL jvp)
vp->SetPrec(jvp->Nd);
break;
default:
case TYPE_DATE:
if (jvp->GetValType() == TYPE_STRG) {
if (!((DTVAL*)vp)->IsFormatted())
((DTVAL*)vp)->SetFormat(g, "YYYY-MM-DDThh:mm:ssZ", 20, 0);
vp->SetValue_psz(jvp->GetString(g));
} else
vp->SetValue(jvp->GetInteger());
break;
default:
sprintf(g->Message, "Unsupported column type %d\n", vp->GetType());
throw 888;
} // endswitch Type
......
storage/connect/tabjson.h
View file @ e85006e6
/*************** tabjson H Declares Source Code File (.H) **************/
/* Name: tabjson.h Version 1.3 */
/* */
/* (C) Copyright to the author Olivier BERTRAND 2014 - 2018 */
/* (C) Copyright to the author Olivier BERTRAND 2014 - 2021 */
/* */
/* This file contains the JSON classes declares. */
/***********************************************************************/
......@@ -67,10 +67,11 @@ class JSONDISC : public BLOCK {
PJSON jsp;
PJOB row;
PCSZ sep;
PCSZ strfy;
char colname[65], fmt[129], buf[16];
uint *length;
int i, n, bf, ncol, lvl, sz, limit;
bool all, strfy;
bool all;
}; // end of JSONDISC
/***********************************************************************/
......
storage/connect/tabrest.cpp
View file @ e85006e6
......@@ -99,12 +99,12 @@ int Xcurl(PGLOBAL g, PCSZ Http, PCSZ Uri, PCSZ filename)
if (Uri) {
if (*Uri == '/' || Http[strlen(Http) - 1] == '/')
sprintf(buf, "curl %s%s -o %s", Http, Uri, filename);
sprintf(buf, "curl \"%s%s\" -o %s", Http, Uri, filename);
else
sprintf(buf, "curl %s/%s -o %s", Http, Uri, filename);
sprintf(buf, "curl \"%s/%s\" -o %s", Http, Uri, filename);
} else
sprintf(buf, "curl %s -o %s", Http, filename);
sprintf(buf, "curl \"%s\" -o %s", Http, filename);
if ((pipe = popen(buf, "rt"))) {
if (trace(515))
......@@ -202,11 +202,11 @@ PQRYRES __stdcall ColREST(PGLOBAL g, PTOS tp, char *tab, char *db, bool info)
PQRYRES qrp= NULL;
char filename[_MAX_PATH + 1]; // MAX PATH ???
int rc;
bool curl = false;
PCSZ http, uri, fn, ftype;
XGETREST grf = GetRestFunction(g);
XGETREST grf = NULL;
bool curl = GetBooleanTableOption(g, tp, "Curl", false);
if (!grf)
if (!curl && !(grf = GetRestFunction(g)))
curl = true;
http = GetStringTableOption(g, tp, "Http", NULL);
......@@ -230,28 +230,25 @@ PQRYRES __stdcall ColREST(PGLOBAL g, PTOS tp, char *tab, char *db, bool info)
filename[n + i] = tolower(ftype[i]);
fn = filename;
tp->filename = PlugDup(g, fn);
tp->subtype = PlugDup(g, fn);
sprintf(g->Message, "No file name. Table will use %s", fn);
PUSH_WARNING(g->Message);
} // endif fn
// We used the file name relative to recorded datapath
PlugSetPath(filename, fn, db);
curl = GetBooleanTableOption(g, tp, "Curl", curl);
remove(filename);
// Retrieve the file from the web and copy it locally
if (curl)
rc = Xcurl(g, http, uri, filename);
else if (grf)
else
rc = grf(g->Message, trace(515), http, uri, filename);
else {
strcpy(g->Message, "Cannot access to curl nor casablanca");
rc = 1;
} // endif !grf
if (rc)
if (rc) {
strcpy(g->Message, "Cannot access to curl nor casablanca");
return NULL;
else if (!stricmp(ftype, "JSON"))
} else if (!stricmp(ftype, "JSON"))
qrp = JSONColumns(g, db, NULL, tp, info);
else if (!stricmp(ftype, "CSV"))
qrp = CSVColumns(g, NULL, tp, info);
......@@ -274,11 +271,12 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff)
{
char filename[_MAX_PATH + 1];
int rc = 0, n;
bool curl = false, xt = trace(515);
bool xt = trace(515);
LPCSTR ftype;
XGETREST grf = GetRestFunction(g);
XGETREST grf = NULL;
bool curl = GetBoolCatInfo("Curl", false);
if (!grf)
if (!curl && !(grf = GetRestFunction(g)))
curl = true;
#if defined(MARIADB)
......@@ -309,24 +307,21 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff)
// We used the file name relative to recorded datapath
PlugSetPath(filename, Fn, GetPath());
curl = GetBoolCatInfo("Curl", curl);
remove(filename);
// Retrieve the file from the web and copy it locally
if (curl) {
rc = Xcurl(g, Http, Uri, filename);
xtrc(515, "Return from Xcurl: rc=%d\n", rc);
} else if (grf) {
} else {
rc = grf(g->Message, xt, Http, Uri, filename);
xtrc(515, "Return from restGetFile: rc=%d\n", rc);
} else {
strcpy(g->Message, "Cannot access to curl nor casablanca");
rc = 1;
} // endif !grf
} // endelse
if (rc)
return true;
else switch (n) {
if (rc) {
strcpy(g->Message, "Cannot access to curl nor casablanca");
return true;
} else switch (n) {
case 1: Tdp = new (g) JSONDEF; break;
#if defined(XML_SUPPORT)
case 2: Tdp = new (g) XMLDEF; break;
......
storage/connect/tabxml.cpp
View file @ e85006e6
......@@ -148,14 +148,21 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info)
/* Open the input file. */
/*********************************************************************/
if (!(fn = GetStringTableOption(g, topt, "Filename", NULL))) {
strcpy(g->Message, MSG(MISSING_FNAME));
return NULL;
} else {
lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth());
lvl = GetIntegerTableOption(g, topt, "Depth", lvl);
lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl;
if (topt->http) // REST table can have default filename
fn = GetStringTableOption(g, topt, "Subtype", NULL);
if (!fn) {
strcpy(g->Message, MSG(MISSING_FNAME));
return NULL;
} else
topt->subtype = NULL;
} // endif fn
lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth());
lvl = GetIntegerTableOption(g, topt, "Depth", lvl);
lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl;
if (trace(1))
htrc("File %s lvl=%d\n", topt->filename, lvl);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7