From: Matt Birkholz <puck@birchwood-abbey.net>
Date: Sat, 20 Dec 2014 15:37:56 +0000 (-0700)
Subject: smp: Synchronize initialized processors.
X-Git-Url: https://birchwood-abbey.net/git?a=commitdiff_plain;h=e25d80c78040fbf038e20956bfd052a62352a456;p=mit-scheme.git

smp: Synchronize initialized processors.

The main pthread must wait for the other processors to be initialized,
else the first GC waits forever for processors that cleared the
global-gc interrupt while initializing.
---

diff --git a/src/microcode/boot.c b/src/microcode/boot.c
index 3fccf6dd9..555d90fe3 100644
--- a/src/microcode/boot.c
+++ b/src/microcode/boot.c
@@ -126,12 +126,15 @@ main_name (int argc, const char ** argv)
 		option_processor_count,
 		(BLOCKS_TO_BYTES (option_processor_heap_size)),
 		(BLOCKS_TO_BYTES (option_stack_size)));
-  smp_initialize (option_processor_count);
+  setup_processors (option_processor_count);
 #endif
 
   initialize_primitives ();
   compiler_initialize (option_fasl_file != 0);
   OS_initialize ();
+#ifdef ENABLE_SMP
+  start_processors ();
+#endif
   start_scheme ();
   termination_init_error ();
   return (0);
diff --git a/src/microcode/ossmp.h b/src/microcode/ossmp.h
index 449687caf..cf09e16d9 100644
--- a/src/microcode/ossmp.h
+++ b/src/microcode/ossmp.h
@@ -34,7 +34,7 @@ USA.
 typedef enum {
   PROCESSOR_NEW,
   PROCESSOR_RUNNING,
-  PROCESSOR_PAUSED,
+  PROCESSOR_IDLE,
   PROCESSOR_GC_WAIT,
   PROCESSOR_DEAD
 } processor_state_t;
@@ -61,7 +61,8 @@ struct processor {
 extern processor_t *processors;
 extern __thread processor_t *self;
 
-extern void smp_initialize (int processor_count);
+extern void setup_processors (int count);
+extern void start_processors (void);
 
 extern void smp_gc_start (void);
 extern bool smp_gc_started (void);
diff --git a/src/microcode/prossmp.c b/src/microcode/prossmp.c
index 944224d46..c47ebdb3a 100644
--- a/src/microcode/prossmp.c
+++ b/src/microcode/prossmp.c
@@ -33,6 +33,7 @@ USA.
 #include "history.h"
 #include "osio.h"
 #include "ossmp.h"
+#include "osproc.h"	/* for OS_process_any_status_change */
 #include <pthread.h>
 
 /* The chain of processors, starting with processor0 -- main()'s thread: */
@@ -193,7 +194,7 @@ thread switches.")
 static void make_processors (int);
 
 void
-smp_initialize (int count)
+setup_processors (int count)
 {
   make_processors (count-1);
 
@@ -272,9 +273,6 @@ make_processors (int id)
   new->heap_end = heap_start + saved_processor_heap_size;
   processors = new;
 
-  if (id != 0)
-    create (id, &new->pthread, &work, new);
-
   trace (";%d heap:  0x%0lx-0x%0lx", id,
 	 (ulong)new->heap_start, (ulong)new->heap_end);
   trace (";%d stack: 0x%0lx-0x%0lx", id,
@@ -284,6 +282,29 @@ make_processors (int id)
     make_processors (id - 1);
 }
 
+static bool all_in (processor_state_t s);
+
+void
+start_processors (void)
+{
+  processor_t *p;
+
+  for (p = processors->next; p != NULL; p = p->next)
+    create (p->id, &p->pthread, &work, p);
+
+  mutex_lock (&state_mutex);
+  self->state = PROCESSOR_GC_WAIT;
+  trace (";0 Pausing.");
+  if (! all_in (PROCESSOR_GC_WAIT))
+    {
+      trace (";0 Waiting (for all to pause).");
+      cond_wait (&ready, &state_mutex);
+    }
+  trace (";0 Ready (all paused).");
+  self->state = PROCESSOR_RUNNING;
+  mutex_unlock (&state_mutex);
+}
+
 static bool
 all_in (processor_state_t s)
 {
@@ -436,8 +457,9 @@ smp_gc_finished (void)
   cond_broadcast (&finished);
   mutex_unlock (&state_mutex);
 }
-#endif
 
+static SCHEME_OBJECT get_smp_idle_prim (void);
+#endif
 
 DEFINE_PRIMITIVE ("SMP-PAUSE", Prim_smp_pause, 0, 0, "(SMP-PAUSE)\n\
 Pause a new processor.")
@@ -445,15 +467,70 @@ Pause a new processor.")
   PRIMITIVE_HEADER (0);
 #ifdef ENABLE_SMP
   assert (self->id != 0);
-  self->state = PROCESSOR_PAUSED;
-  OS_pause ();
+
+ Will_Push (STACK_ENV_EXTRA_SLOTS + 1);
+  STACK_PUSH (get_smp_idle_prim ());
+  PUSH_APPLY_FRAME_HEADER (0);
+ Pushed ();
+
+  SET_INTERRUPT_MASK (INT_Mask);
+  PRIMITIVE_ABORT (PRIM_APPLY);
+#else
+  signal_error_from_primitive (ERR_UNIMPLEMENTED_PRIMITIVE);
+#endif
+}
+
+DEFINE_PRIMITIVE ("SMP-IDLE", Prim_smp_idle, 0, 0, "(SMP-IDLE)\n\
+Wait for interrupts.")
+{
+  PRIMITIVE_HEADER (0);
+
+#ifdef ENABLE_SMP
+  assert (self->state == PROCESSOR_RUNNING || self->state == PROCESSOR_IDLE);
+
+  trace (";%d SMP-Idle.", self->id);
+  self->state = PROCESSOR_IDLE;
+
+  assert (GET_INT_MASK == INT_Mask);
+  while (! ((PENDING_INTERRUPTS_P)
+	    || OS_process_any_status_change ()))
+    {
+      OS_pause ();
+      trace (";%d SMP-Idle awoke to 0x%04x.", self->id, GET_INT_CODE);
+    }
+
   self->state = PROCESSOR_RUNNING;
+  if (OS_process_any_status_change ())
+    {
+      outf_error_line (";%d synthetic timer interrupt used", self->id);
+      REQUEST_INTERRUPT (INT_Timer); /* Wake to handle (maybe) a SIGCHLD. */
+    }
+  assert (PENDING_INTERRUPTS_P);
+  signal_interrupt_from_primitive ();
 #else
   signal_error_from_primitive (ERR_UNIMPLEMENTED_PRIMITIVE);
 #endif
-  PRIMITIVE_RETURN (UNSPECIFIC);
 }
 
+#ifdef ENABLE_SMP
+static SCHEME_OBJECT smp_idle_prim = SHARP_F;
+
+static SCHEME_OBJECT
+get_smp_idle_prim (void)
+{
+  if (smp_idle_prim != SHARP_F)
+    return (smp_idle_prim);
+
+  smp_idle_prim = find_primitive_cname ("SMP-IDLE", false, false, 0);
+  if (smp_idle_prim == SHARP_F)
+    {
+      outf_error_line (";No SMP-IDLE primitive!");
+      signal_error_from_primitive (ERR_EXTERNAL_RETURN);
+    }
+  return (smp_idle_prim);
+}
+#endif
+
 DEFINE_PRIMITIVE ("SMP-GC-WAIT", Prim_smp_gc_wait, 0, 0, "(SMP-GC-WAIT)\n\
 Put the current processor in the GC-WAIT state.\n\
 Called by the global-gc interrupt handler.")