From d32525d1709fc816abf0896c239caf86cf25d5ca Mon Sep 17 00:00:00 2001
From: Matt Birkholz <puck@birchwood-abbey.net>
Date: Tue, 18 Aug 2015 09:35:16 -0700
Subject: [PATCH] Synchronize initialized processors.

The main pthread must wait for the other processors to be initialized,
else the first smp-gc-lock waits forever for processors that cleared
their global-gc interrupt while initializing.
---
 src/microcode/boot.c    |   8 +++
 src/microcode/ossmp.h   |   1 +
 src/microcode/prossmp.c | 115 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 124 insertions(+)

diff --git a/src/microcode/boot.c b/src/microcode/boot.c
index 9d815e52f..59ba6c643 100644
--- a/src/microcode/boot.c
+++ b/src/microcode/boot.c
@@ -132,6 +132,11 @@ main_name (int argc, const char ** argv)
   initialize_primitives ();
   compiler_initialize (option_fasl_file != 0);
   OS_initialize ();
+#ifdef ENABLE_SMP
+  start_processors ();
+  assert (gc_processor == NULL);
+  gc_processor = self;
+#endif
   start_scheme ();
   termination_init_error ();
   return (0);
@@ -177,6 +182,9 @@ start_scheme (void)
       (*Free++) = prim2;
       (*Free++) = (MAKE_POINTER_OBJECT (TC_COMBINATION, inner_arg));
       (*Free++) = THE_GLOBAL_ENV;
+#ifdef ENABLE_SMP
+      gc_processor = NULL;
+#endif
     }
   else
     {
diff --git a/src/microcode/ossmp.h b/src/microcode/ossmp.h
index 61570364c..7f074a277 100644
--- a/src/microcode/ossmp.h
+++ b/src/microcode/ossmp.h
@@ -60,6 +60,7 @@ extern __thread processor_t *self;
 extern processor_t *gc_processor;
 
 extern void setup_processors (int count);
+extern void start_processors (void);
 
 extern void smp_gc_start (void);
 extern void smp_gc_finish (void);
diff --git a/src/microcode/prossmp.c b/src/microcode/prossmp.c
index 17fb1702c..6bb134a32 100644
--- a/src/microcode/prossmp.c
+++ b/src/microcode/prossmp.c
@@ -145,6 +145,17 @@ cond_broadcast (pthread_cond_t *cond)
     }
 }
 
+static void
+create (int id, pthread_t *thread, void *(*start_routine) (void *), void *arg)
+{
+  int err = pthread_create (thread, NULL, start_routine, arg);
+  if (err != 0)
+    {
+      pthread_error (err);
+      fatal ("\n;%d pthread_create failed: %d", id, err);
+    }
+}
+
 static void
 pthread_error (int code)
 {
@@ -182,6 +193,45 @@ setup_processors (int count)
   RESET_HISTORY ();
 }
 
+static void *
+work (void *p)
+{
+  SCHEME_OBJECT expr;
+  self = (processor_t *)p;
+  trace (";%d Start.", self->id);
+  assert (self->id != 0);
+  assert (self != processors);
+  reset_processor_allocator (self);
+  RESET_HISTORY ();
+
+  /* expr: (SMP-PAUSE) */
+  expr = (MAKE_POINTER_OBJECT (TC_COMBINATION, Free));
+  (*Free++) = MAKE_OBJECT (TC_MANIFEST_VECTOR, 1);
+  (*Free++) = make_primitive ("SMP-PAUSE", 0);
+
+  /* Setup registers */
+  INITIALIZE_INTERRUPTS (0);
+  SET_ENV (THE_GLOBAL_ENV);
+  trapping = false;
+
+  /* Give the interpreter something to chew on, and ... */
+  Will_Push (CONTINUATION_SIZE);
+  SET_RC (RC_END_OF_COMPUTATION);
+  SET_EXP (SHARP_F);
+  SAVE_CONT ();
+  Pushed ();
+
+  SET_EXP (expr);
+
+  /* Go to it! */
+
+  Interpret (0);
+
+  trace (";%d Died.", self->id);
+  self->state = PROCESSOR_DEAD;
+  return (NULL);
+}
+
 static void
 make_processors (int id)
 {
@@ -217,6 +267,25 @@ make_processors (int id)
     make_processors (id - 1);
 }
 
+static bool all_in (processor_state_t s);
+
+void
+start_processors (void)
+{
+  for (processor_t *p = processors->next; p != NULL; p = p->next)
+    create (p->id, &p->pthread, &work, p);
+
+  mutex_lock (&state_mutex);
+  self->state = PROCESSOR_GC_WAIT;
+  if (! all_in (PROCESSOR_GC_WAIT))
+    {
+      trace (";0 Pausing.");
+      cond_wait (&ready, &state_mutex);
+    }
+  trace (";0 Paused.");
+  mutex_unlock (&state_mutex);
+}
+
 static bool
 all_in (processor_state_t s)
 {
@@ -300,6 +369,23 @@ smp_gc_finish (void)
       p->free_pointer = p->heap_start;
   trace (";%d smp_gc_finish reset", self->id);
 }
+
+static SCHEME_OBJECT smp_idle_prim = SHARP_F;
+
+static SCHEME_OBJECT
+get_smp_idle_prim (void)
+{
+  if (smp_idle_prim != SHARP_F)
+    return (smp_idle_prim);
+
+  smp_idle_prim = find_primitive_cname ("SMP-IDLE", false, false, 0);
+  if (smp_idle_prim == SHARP_F)
+    {
+      outf_error_line (";No SMP-IDLE primitive!");
+      signal_error_from_primitive (ERR_EXTERNAL_RETURN);
+    }
+  return (smp_idle_prim);
+}
 #endif /* ENABLE_SMP */
 
 DEFINE_PRIMITIVE ("SMP-COUNT", Prim_smp_count, 0, 0, "(SMP-COUNT)\n\
@@ -360,6 +446,35 @@ access to the thread system.")
 #endif
 }
 
+DEFINE_PRIMITIVE ("SMP-PAUSE", Prim_smp_pause, 0, 0, "(SMP-PAUSE)\n\
+Pause a new processor.")
+{
+  PRIMITIVE_HEADER (0);
+#ifdef ENABLE_SMP
+  assert (self->id != 0);
+  while (compiler_utilities == SHARP_F)
+    {
+      trace (";%d SMP-Pause.", self->id);
+      mutex_lock (&state_mutex);
+      gc_wait ();
+      mutex_unlock (&state_mutex);
+      trace (";%d SMP-Pause done.", self->id);
+      /* If no compiler_utilities, then not bootstrapping; must wait
+	 for band-load. */
+    }
+
+ Will_Push (STACK_ENV_EXTRA_SLOTS + 1);
+  STACK_PUSH (get_smp_idle_prim ());
+  PUSH_APPLY_FRAME_HEADER (0);
+ Pushed ();
+
+  SET_INTERRUPT_MASK (INT_Mask);
+  PRIMITIVE_ABORT (PRIM_APPLY);
+#else
+  signal_error_from_primitive (ERR_UNIMPLEMENTED_PRIMITIVE);
+#endif
+}
+
 DEFINE_PRIMITIVE ("SMP-IDLE", Prim_smp_idle, 0, 0, "(SMP-IDLE)\n\
 Wait for interrupts.")
 {
-- 
2.25.1