Diff of /src/runtime/win32-os.c [3ea89b] .. [f05756] Maximize Restore

  Switch to side-by-side view

--- a/src/runtime/win32-os.c
+++ b/src/runtime/win32-os.c
@@ -247,6 +247,8 @@
 
 void *base_seh_frame;
 
+HMODULE runtime_module_handle = 0u;
+
 static void *get_seh_frame(void)
 {
     void* retval;
@@ -311,6 +313,302 @@
 }
 
 #endif
+
+#if defined(LISP_FEATURE_SB_DYNAMIC_CORE)
+/* This feature has already saved me more development time than it
+ * took to implement.  In its current state, ``dynamic RT<->core
+ * linking'' is a protocol of initialization of C runtime and Lisp
+ * core, populating SBCL linkage table with entries for runtime
+ * "foreign" symbols that were referenced in cross-compiled code.
+ *
+ * How it works: a sketch
+ *
+ * Last Genesis (resulting in cold-sbcl.core) binds foreign fixups in
+ * x-compiled lisp-objs to sequential addresses from the beginning of
+ * linkage-table space; that's how it ``resolves'' foreign references.
+ * Obviously, this process doesn't require pre-built runtime presence.
+ *
+ * When the runtime loads the core (cold-sbcl.core initially,
+ * sbcl.core later), runtime should do its part of the protocol by (1)
+ * traversing a list of ``runtime symbols'' prepared by Genesis and
+ * dumped as a static symbol value, (2) resolving each name from this
+ * list to an address (stubbing unresolved ones with
+ * undefined_alien_address or undefined_alien_function), (3) adding an
+ * entry for each symbol somewhere near the beginning of linkage table
+ * space (location is provided by the core).
+ *
+ * The implementation of the part described in the last paragraph
+ * follows. C side is currently more ``hackish'' and less clear than
+ * the Lisp code; OTOH, related Lisp changes are scattered, and some
+ * of them play part in complex interrelations -- beautiful but taking
+ * much time to understand --- but my subset of PE-i386 parser below
+ * is in one place (here) and doesn't have _any_ non-trivial coupling
+ * with the rest of the Runtime.
+ *
+ * What do we gain with this feature, after all?
+ *
+ * One things that I have to do rather frequently: recompile and
+ * replace runtime without rebuilding the core. Doubtlessly, slam.sh
+ * was a great time-saver here, but relinking ``cold'' core and bake a
+ * ``warm'' one takes, as it seems, more than 10x times of bare
+ * SBCL.EXE build time -- even if everything is recompiled, which is
+ * now unnecessary. Today, if I have a new idea for the runtime,
+ * getting from C-x C-s M-x ``compile'' to fully loaded SBCL
+ * installation takes 5-15 seconds.
+ *
+ * Another thing (that I'm not currently using, but obviously
+ * possible) is delivering software patches to remote system on
+ * customer site. As you are doing minor additions or corrections in
+ * Lisp code, it doesn't take much effort to prepare a tiny ``FASL
+ * bundle'' that rolls up your patch, redumps and -- presto -- 100MiB
+ * program is fixed by sending and loading a 50KiB thingie.
+ *
+ * However, until LISP_FEATURE_SB_DYNAMIC_CORE, if your bug were fixed
+ * by modifying two lines of _C_ sources, a customer described above
+ * had to be ready to receive and reinstall a new 100MiB
+ * executable. With the aid of code below, deploying such a fix
+ * requires only sending ~300KiB (when stripped) of SBCL.EXE.
+ *
+ * But there is more to it: as the common linkage-table is used for
+ * DLLs and core, its entries may be overridden almost without a look
+ * into SBCL internals. Therefore, ``patching'' C runtime _without_
+ * restarting target systems is also possible in many situations
+ * (it's not as trivial as loading FASLs into a running daemon, but
+ * easy enough to be a viable alternative if any downtime is highly
+ * undesirable).
+ *
+ * During my (rather limited) commercial Lisp development experience
+ * I've already been through a couple of situations where such
+ * ``deployment'' issues were important; from my _total_ programming
+ * experience I know -- _sometimes_ they are a two orders of magnitude
+ * more important than those I observed.
+ *
+ * The possibility of entire runtime ``hot-swapping'' in running
+ * process is not purely theoretical, as it could seem. There are 2-3
+ * problems whose solution is not obvious (call stack patching, for
+ * instance), but it's literally _nothing_ if compared with
+ * e.g. LISP_FEATURE_SB_AUTO_FPU_SWITCH.  By the way, one of the
+ * problems with ``hot-swapping'', that could become a major one in
+ * many other environments, is nonexistent in SBCL: we already have a
+ * ``global quiesce point'' that is generally required for this kind
+ * of worldwide revolution -- around collect_garbage.
+ *
+ * What's almost unnoticeable from the C side (where you are now, dear
+ * reader): using the same style for all linking is beautiful. I tried
+ * to leave old-style linking code in place for the sake of
+ * _non-linkage-table_ platforms (they probably don't have -ldl or its
+ * equivalent, like LL/GPA, at all) -- but i did it usually by moving
+ * the entire `old style' code under #!-sb-dynamic-core and
+ * refactoring the `new style' branch, instead of cutting the tail
+ * piecemeal and increasing #!+-ifdeffery amount & the world enthropy.
+ *
+ * If we look at the majority of the ``new style'' code units, it's a
+ * common thing to observe how #!+-ifdeffery _vanishes_ instead of
+ * multiplying: #!-sb-xc, #!+sb-xc-host and #!-sb-xc-host end up
+ * needing the same code. Runtime checks of static v. dynamic symbol
+ * disappear even faster. STDCALL mangling and leading underscores go
+ * out of scope (and GCed, hopefully) instead of surfacing here and
+ * there as a ``special case for core static symbols''. What I like
+ * the most about CL development in general is a frequency of solving
+ * problems and fixing bugs by simplifying code and dropping special
+ * cases.
+ *
+ * Last important thing about the following code: besides resolving
+ * symbols provided by the core itself, it detects runtime's own
+ * build-time prerequisite DLLs. Any symbol that is unresolved against
+ * the core is looked up in those DLLs (normally kernel32, msvcrt,
+ * ws2_32... I could forget something). This action (1) resembles
+ * implementation of foreign symbol lookup in SBCL itself, (2)
+ * emulates shared library d.l. facilities of OSes that use flat
+ * dynamic symbol namespace (or default to it). Anyone concerned with
+ * portability problems of this PE-i386 stuff below will be glad to
+ * hear that it could be ported to most modern Unices _by deletion_:
+ * raw dlsym() with null handle usually does the same thing that i'm
+ * trying to squeeze out of MS Windows by the brute force.
+ *
+ * My reason for _desiring_ flat symbol namespace, populated from
+ * link-time dependencies, is avoiding any kind of ``requested-by-Lisp
+ * symbol lists to be linked statically'', providing core v. runtime
+ * independence in both directions. Minimizing future maintenance
+ * effort is very important; I had gone for it consistently, starting
+ * by turning "CloseHandle@4" into a simple "CloseHandle", continuing
+ * by adding intermediate Genesis resulting in autogenerated symbol
+ * list (farewell, void scratch(); good riddance), going to take
+ * another great step for core/runtime independence... and _without_
+ * flat namespace emulation, the ghosts and spirits exiled at the
+ * first steps would come and take revenge: well, here are the symbols
+ * that are really in msvcrt.dll.. hmm, let's link statically against
+ * them, so the entry is pulled from the import library.. and those
+ * entry has mangled names that we have to map.. ENOUGH, I though
+ * here: fed up with stuff like that.
+ *
+ * Now here we are, without import libraries, without mangled symbols,
+ * and without nm-generated symbol tables. Every symbol exported by
+ * the runtime is added to SBCL.EXE export directory; every symbol
+ * requested by the core is looked up by GetProcAddress for SBCL.EXE,
+ * falling back to GetProcAddress for MSVCRT.dll, etc etc.. All ties
+ * between SBCL's foreign symbols with object file symbol tables,
+ * import libraries and other pre-linking symbol-resolving entities
+ * _having no representation in SBCL.EXE_ were teared.
+ *
+ * This simplistic approach proved to work well; there is only one
+ * problem introduced by it, and rather minor: in real MSVCRT.dll,
+ * what's used to be available as open() is now called _open();
+ * similar thing happened to many other `lowio' functions, though not
+ * every one, so it's not a kind of name mangling but rather someone's
+ * evil creative mind in action.
+ *
+ * When we look up any of those poor `uglified' functions in CRT
+ * reference on MSDN, we can see a notice resembling this one:
+ *
+ * `unixishname()' is obsolete and provided for backward
+ * compatibility; new standard-compliant function, `_unixishname()',
+ * should be used instead.  Sentences of that kind were there for
+ * several years, probably even for a decade or more (a propos,
+ * MSVCRT.dll, as the name to link against, predates year 2000, so
+ * it's actually possible). Reasoning behing it (what MS people had in
+ * mind) always seemed strange to me: if everyone uses open() and that
+ * `everyone' is important to you, why rename the function?  If no one
+ * uses open(), why provide or retain _open() at all? <kidding>After
+ * all, names like _open() are entirely non-informative and just plain
+ * ugly; compare that with CreateFileW() or InitCommonControlsEx(),
+ * the real examples of beauty and clarity.</kidding>
+ *
+ * Anyway, if the /standard/ name on Windows is _open() (I start to
+ * recall, vaguely, that it's because of _underscore names being
+ * `reserved to system' and all other ones `available for user', per
+ * ANSI/ISO C89) -- well, if the /standard/ name is _open, SBCL should
+ * use it when it uses MSVCRT and not some ``backward-compatible''
+ * stuff. Deciding this way, I added a hack to SBCL's syscall macros,
+ * so "[_]open" as a syscall name is interpreted as a request to link
+ * agains "_open" on win32 and "open" on every other system.
+ *
+ * Of course, this name-parsing trick lacks conceptual clarity; we're
+ * going to get rid of it eventually. */
+
+u32 os_get_build_time_shared_libraries(u32 excl_maximum,
+                                       void* opt_root,
+                                       void** opt_store_handles,
+                                       const char *opt_store_names[])
+{
+    void* base = opt_root ? opt_root : (void*)runtime_module_handle;
+    /* base defaults to 0x400000 with GCC/mingw32. If you dereference
+     * that location, you'll see 'MZ' bytes */
+    void* base_magic_location =
+        base + ((IMAGE_DOS_HEADER*)base)->e_lfanew;
+
+    /* dos header provided the offset from `base' to
+     * IMAGE_FILE_HEADER where PE-i386 really starts */
+
+    void* check_duplicates[excl_maximum];
+
+    if ((*(u32*)base_magic_location)!=0x4550) {
+        /* We don't need this DLL thingie _that_ much. If the world
+         * has changed to a degree where PE magic isn't found, let's
+         * silently return `no libraries detected'. */
+        return 0;
+    } else {
+        /* We traverse PE-i386 structures of SBCL.EXE in memory (not
+         * in the file). File and memory layout _surely_ differ in
+         * some places and _may_ differ in some other places, but
+         * fortunately, those places are irrelevant to the task at
+         * hand. */
+
+        IMAGE_FILE_HEADER* image_file_header = (base_magic_location + 4);
+        IMAGE_OPTIONAL_HEADER* image_optional_header =
+            (void*)(image_file_header + 1);
+        IMAGE_DATA_DIRECTORY* image_import_direntry =
+            &image_optional_header->DataDirectory[IMAGE_DIRECTORY_ENTRY_IMPORT];
+        IMAGE_IMPORT_DESCRIPTOR* image_import_descriptor =
+            base + image_import_direntry->VirtualAddress;
+        u32 nlibrary, i,j;
+
+        for (nlibrary=0u; nlibrary < excl_maximum
+                          && image_import_descriptor->FirstThunk;
+             ++image_import_descriptor)
+        {
+            HMODULE hmodule;
+            odxprint(runtime_link, "Now should know DLL: %s",
+                     (char*)(base + image_import_descriptor->Name));
+            /* Code using image thunk data to get its handle was here, with a
+             * number of platform-specific tricks (like using VirtualQuery for
+             * old OSes lacking GetModuleHandleEx).
+             *
+             * It's now replaced with requesting handle by name, which is
+             * theoretically unreliable (with SxS, multiple modules with same
+             * name are quite possible), but good enough to find the
+             * link-time dependencies of our executable or DLL. */
+
+            hmodule = (HMODULE)
+                GetModuleHandle(base + image_import_descriptor->Name);
+
+            if (hmodule)
+            {
+                /* We may encouncer some module more than once while
+                   traversing import descriptors (it's usually a
+                   result of non-trivial linking process, like doing
+                   ld -r on some groups of files before linking
+                   everything together.
+
+                   Anyway: using a module handle more than once will
+                   do no harm, but it slows down the startup (even
+                   now, our startup time is not a pleasant topic to
+                   discuss when it comes to :sb-dynamic-core; there is
+                   an obvious direction to go for speed, though --
+                   instead of resolving symbols one-by-one, locate PE
+                   export directories -- they are sorted by symbol
+                   name -- and merge them, at one pass, with sorted
+                   list of required symbols (the best time to sort the
+                   latter list is during Genesis -- that's why I don't
+                   proceed with memory copying, qsort() and merge
+                   right here)). */
+
+                for (j=0; j<nlibrary; ++j)
+                {
+                    if(check_duplicates[j] == hmodule)
+                        break;
+                }
+                if (j<nlibrary) continue; /* duplicate => skip it in
+                                           * outer loop */
+
+                check_duplicates[nlibrary] = hmodule;
+                if (opt_store_handles) {
+                    opt_store_handles[nlibrary] = hmodule;
+                }
+                if (opt_store_names) {
+                    opt_store_names[nlibrary] = (const char *)
+                        (base + image_import_descriptor->Name);
+                }
+                odxprint(runtime_link, "DLL detection: %u, base %p: %s",
+                         nlibrary, hmodule,
+                         (char*)(base + image_import_descriptor->Name));
+                ++ nlibrary;
+            }
+        }
+        return nlibrary;
+    }
+}
+
+static u32 buildTimeImageCount = 0;
+static void* buildTimeImages[16];
+
+/* Resolve symbols against the executable and its build-time dependencies */
+void* os_dlsym_default(char* name)
+{
+    unsigned int i;
+    void* result = 0;
+    if (buildTimeImageCount == 0) {
+        buildTimeImageCount =
+            1 + os_get_build_time_shared_libraries(15u,
+            NULL, 1+(void**)buildTimeImages, NULL);
+    }
+    for (i = 0; i<buildTimeImageCount && (!result); ++i) {
+        result = GetProcAddress(buildTimeImages[i], name);
+    }
+    return result;
+}
+
+#endif /* SB_DYNAMIC_CORE */
 
 #if defined(LISP_FEATURE_SB_THREAD)
 /* We want to get a slot in TIB that (1) is available at constant
@@ -406,6 +704,20 @@
 
 #undef RESOLVE
 
+intptr_t win32_get_module_handle_by_address(os_vm_address_t addr)
+{
+    HMODULE result = 0;
+    /* So apparently we could use VirtualQuery instead of
+     * GetModuleHandleEx if we wanted to support pre-XP, pre-2003
+     * versions of Windows (i.e. Windows 2000).  I've opted against such
+     * special-casing. :-).  --DFL */
+    return (intptr_t)(GetModuleHandleEx(
+                          GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
+                          GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
+                          (LPCSTR)addr, &result)
+                      ? result : 0);
+}
+
 void os_init(char *argv[], char *envp[])
 {
     SYSTEM_INFO system_info;
@@ -420,6 +732,7 @@
     base_seh_frame = get_seh_frame();
 
     resolve_optional_imports();
+    runtime_module_handle = (HMODULE)win32_get_module_handle_by_address(&runtime_module_handle);
 }
 
 static inline boolean local_thread_stack_address_p(os_vm_address_t address)
@@ -552,6 +865,19 @@
     AVERLAX(VirtualQuery(addr, &minfo, sizeof minfo));
     AVERLAX(minfo.AllocationBase);
     AVERLAX(VirtualFree(minfo.AllocationBase, 0, MEM_RELEASE));
+}
+
+/* os_validate doesn't commit, i.e. doesn't actually "validate" in the
+ * sense that we could start using the space afterwards.  Usually it's
+ * os_map or Lisp code that will run into that, in which case we recommit
+ * elsewhere in this file.  For cases where C wants to write into newly
+ * os_validate()d memory, it needs to commit it explicitly first:
+ */
+os_vm_address_t
+os_validate_recommit(os_vm_address_t addr, os_vm_size_t len)
+{
+    return
+        AVERLAX(VirtualAlloc(addr, len, MEM_COMMIT, PAGE_EXECUTE_READWRITE));
 }
 
 #define maybe_open_osfhandle _open_osfhandle
@@ -1685,84 +2011,26 @@
     return read_bytes;
 }
 
-/* This is a manually-maintained version of ldso_stubs.S. */
-
 void __stdcall RtlUnwind(void *, void *, void *, void *); /* I don't have winternl.h */
 
+/* We used to have a scratch() function listing all symbols needed by
+ * Lisp.  Much rejoicing commenced upon its removal.  However, I would
+ * like cold init to fail aggressively when encountering unused symbols.
+ * That poses a problem, however, since our C code no longer includes
+ * any references to symbols in ws2_32.dll, and hence the linker
+ * completely ignores our request to reference it (--no-as-needed does
+ * not work).  Warm init would later load the DLLs explicitly, but then
+ * it's too late for an early sanity check.  In the unfortunate spirit
+ * of scratch(), continue to reference some required DLLs explicitly by
+ * means of one scratch symbol per DLL.
+ */
 void scratch(void)
 {
-    LARGE_INTEGER la = {{0}};
-    closesocket(0);
-    CloseHandle(0);
+    /* a function from ws2_32.dll */
     shutdown(0, 0);
-    SetHandleInformation(0, 0, 0);
-    GetHandleInformation(0, 0);
-    getsockopt(0, 0, 0, 0, 0);
-    FlushConsoleInputBuffer(0);
-    FormatMessageA(0, 0, 0, 0, 0, 0, 0);
-    FreeLibrary(0);
-    GetACP();
-    GetConsoleCP();
-    GetConsoleOutputCP();
-    GetCurrentProcess();
-    GetExitCodeProcess(0, 0);
-    GetLastError();
-    GetOEMCP();
-    GetProcAddress(0, 0);
-    GetProcessTimes(0, 0, 0, 0, 0);
-    GetSystemTimeAsFileTime(0);
-    LoadLibrary(0);
-    LocalFree(0);
-    PeekConsoleInput(0, 0, 0, 0);
-    PeekNamedPipe(0, 0, 0, 0, 0, 0);
-    ReadFile(0, 0, 0, 0, 0);
-    Sleep(0);
-    WriteFile(0, 0, 0, 0, 0);
-    _get_osfhandle(0);
-    _open_osfhandle(0, 0);
-    _rmdir(0);
-    _pipe(0,0,0);
-    access(0,0);
-    close(0);
-    dup(0);
-    isatty(0);
-    strerror(42);
-    write(0, 0, 0);
-    RtlUnwind(0, 0, 0, 0);
-    MapViewOfFile(0,0,0,0,0);
-    UnmapViewOfFile(0);
-    FlushViewOfFile(0,0);
-    SetFilePointerEx(0, la, 0, 0);
-    DuplicateHandle(0, 0, 0, 0, 0, 0, 0);
-    #ifndef LISP_FEATURE_SB_UNICODE
-      CreateDirectoryA(0,0);
-      CreateFileMappingA(0,0,0,0,0,0);
-      CreateFileA(0,0,0,0,0,0,0);
-      GetComputerNameA(0, 0);
-      GetCurrentDirectoryA(0,0);
-      GetEnvironmentVariableA(0, 0, 0);
-      GetFileAttributesA(0);
-      GetVersionExA(0);
-      MoveFileA(0,0);
-      SHGetFolderPathA(0, 0, 0, 0, 0);
-      SetCurrentDirectoryA(0);
-      SetEnvironmentVariableA(0, 0);
-    #else
-      CreateDirectoryW(0,0);
-      CreateFileMappingW(0,0,0,0,0,0);
-      CreateFileW(0,0,0,0,0,0,0);
-      FormatMessageW(0, 0, 0, 0, 0, 0, 0);
-      GetComputerNameW(0, 0);
-      GetCurrentDirectoryW(0,0);
-      GetEnvironmentVariableW(0, 0, 0);
-      GetFileAttributesW(0);
-      GetVersionExW(0);
-      MoveFileW(0,0);
-      SHGetFolderPathW(0, 0, 0, 0, 0);
-      SetCurrentDirectoryW(0);
-      SetEnvironmentVariableW(0, 0);
-    #endif
-    _exit(0);
+
+    /* a function from shell32.dll */
+    SHGetFolderPathA(0, 0, 0, 0, 0);
 }
 
 char *