Parser fix: call fclose

Add a FileHandle class for c++-style raii on FILE* objects.

Also, add support for using the cpython parser for parsing
strings (still only in -X mode).

src/codegen/parser.cpp

@@ -1018,13 +1018,27 @@ static std::string getParserCommandLine(const char* fn) {
 AST_Module* parse_string(const char* code, FutureFlags inherited_flags) {
     inherited_flags &= ~(CO_NESTED | CO_FUTURE_DIVISION);
 
+    if (ENABLE_CPYTHON_PARSER) {
+        RELEASE_ASSERT(!inherited_flags, "unimplemented");
+        PyCompilerFlags cf;
+        cf.cf_flags = 0;
+        ArenaWrapper arena;
+        assert(arena);
+        const char* fn = "<string>";
+        mod_ty mod = PyParser_ASTFromString(code, fn, Py_file_input, &cf, arena);
+        if (!mod)
+            throwCAPIException();
+        auto rtn = cpythonToPystonAST(mod, fn);
+        return rtn;
+    }
+
     if (ENABLE_PYPA_PARSER || inherited_flags) {
         AST_Module* rtn = pypa_parse_string(code, inherited_flags);
         RELEASE_ASSERT(rtn, "unknown parse error (possibly: '%s'?)", strerror(errno));
         return rtn;
     }
 
-    ASSERT(!inherited_flags, "the old cpython parser doesn't support specifying initial future flags");
+    RELEASE_ASSERT(!inherited_flags, "the old cpython parser doesn't support specifying initial future flags");
 
     int size = strlen(code);
     char buf[] = "pystontmp_XXXXXX";
@@ -1035,10 +1049,11 @@ AST_Module* parse_string(const char* code, FutureFlags inherited_flags) {
         printf("writing %d bytes to %s\n", size, tmp.c_str());
     }
 
-    FILE* f = fopen(tmp.c_str(), "w");
-    fwrite(code, 1, size, f);
-    fputc('\n', f);
-    fclose(f);
+    {
+        FileHandle f(tmp.c_str(), "w");
+        fwrite(code, 1, size, f);
+        fputc('\n', f);
+    }
 
     AST_Module* m = parse_file(tmp.c_str(), inherited_flags);
     removeDirectoryIfExists(tmpdir);
@@ -1050,8 +1065,9 @@ AST_Module* parse_file(const char* fn, FutureFlags inherited_flags) {
     Timer _t("parsing");
 
     if (ENABLE_CPYTHON_PARSER) {
-        ASSERT(!inherited_flags, "unimplemented");
-        FILE* fp = fopen(fn, "r");
+        RELEASE_ASSERT(!inherited_flags, "unimplemented");
+
+        FileHandle fp(fn, "r");
         PyCompilerFlags cf;
         cf.cf_flags = 0;
         ArenaWrapper arena;
@@ -1106,7 +1122,7 @@ const char* getMagic() {
 static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, AST_Module*& module,
                                   FutureFlags inherited_flags) {
     inherited_flags &= ~(CO_NESTED | CO_FUTURE_DIVISION);
-    FILE* cache_fp = fopen(cache_fn.c_str(), "w");
+    FileHandle cache_fp(cache_fn.c_str(), "w");
 
     if (DEBUG_PARSING) {
         fprintf(stderr, "_reparse('%s', '%s'), pypa=%d\n", fn, cache_fn.c_str(), ENABLE_PYPA_PARSER);
@@ -1137,8 +1153,9 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
 
     if (ENABLE_CPYTHON_PARSER || ENABLE_PYPA_PARSER || inherited_flags) {
         if (ENABLE_CPYTHON_PARSER) {
-            ASSERT(!inherited_flags, "unimplemented");
-            FILE* fp = fopen(fn, "r");
+            RELEASE_ASSERT(!inherited_flags, "unimplemented");
+
+            FileHandle fp(fn, "r");
             PyCompilerFlags cf;
             cf.cf_flags = 0;
             ArenaWrapper arena;
@@ -1159,7 +1176,7 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
         checksum = p.second;
         bytes_written += p.first;
     } else {
-        ASSERT(!inherited_flags, "the old cpython parser doesn't support specifying initial future flags");
+        RELEASE_ASSERT(!inherited_flags, "the old cpython parser doesn't support specifying initial future flags");
         FILE* parser = popen(getParserCommandLine(fn).c_str(), "r");
         char buf[80];
         while (true) {
@@ -1188,8 +1205,6 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
         fwrite(&checksum, 1, CHECKSUM_LENGTH, cache_fp);
     memcpy(&file_data[checksum_start + LENGTH_LENGTH], &checksum, CHECKSUM_LENGTH);
 
-    if (cache_fp)
-        fclose(cache_fp);
     return std::move(file_data);
 }
 
@@ -1221,7 +1236,8 @@ AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags) {
                           && cache_stat.st_mtim.tv_nsec > source_stat.st_mtim.tv_nsec))) {
         oss << "reading pyc file\n";
         char buf[1024];
-        FILE* cache_fp = fopen(cache_fn.c_str(), "r");
+
+        FileHandle cache_fp(cache_fn.c_str(), "r");
         if (cache_fp) {
             while (true) {
                 int read = fread(buf, 1, 1024, cache_fp);
@@ -1239,7 +1255,6 @@ AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags) {
                     break;
                 }
             }
-            fclose(cache_fp);
         }
     }
 

src/core/types.h

@@ -803,6 +803,20 @@ public:
     operator PyArena*() const { return arena; }
 };
 
+class FileHandle {
+private:
+    FILE* file;
+
+public:
+    FileHandle(const char* fn, const char* mode) : file(fopen(fn, mode)) {}
+    ~FileHandle() {
+        if (file)
+            fclose(file);
+    }
+
+    operator FILE*() const { return file; }
+};
+
 // similar to Java's Array.binarySearch:
 // return values are either:
 //   >= 0 : the index where a given item was found

src/runtime/capi.cpp

@@ -1367,6 +1367,31 @@ cleanup:
 #endif
 
 extern "C" grammar _PyParser_Grammar;
+
+/* Preferred access to parser is through AST. */
+extern "C" mod_ty PyParser_ASTFromString(const char* s, const char* filename, int start, PyCompilerFlags* flags,
+                                         PyArena* arena) noexcept {
+    mod_ty mod;
+    PyCompilerFlags localflags;
+    perrdetail err;
+    int iflags = PARSER_FLAGS(flags);
+
+    node* n = PyParser_ParseStringFlagsFilenameEx(s, filename, &_PyParser_Grammar, start, &err, &iflags);
+    if (flags == NULL) {
+        localflags.cf_flags = 0;
+        flags = &localflags;
+    }
+    if (n) {
+        flags->cf_flags |= iflags & PyCF_MASK;
+        mod = PyAST_FromNode(n, flags, filename, arena);
+        PyNode_Free(n);
+        return mod;
+    } else {
+        err_input(&err);
+        return NULL;
+    }
+}
+
 extern "C" mod_ty PyParser_ASTFromFile(FILE* fp, const char* filename, int start, char* ps1, char* ps2,
                                        PyCompilerFlags* flags, int* errcode, PyArena* arena) noexcept {
     mod_ty mod;