Configure prodigy

emacs/.emacs.d/config/init-lib.el

@@ -1,4 +1,26 @@
 ;; -*- lexical-binding: t; -*-
+;; Helper packages
+(use-package deferred
+  :defer t)
+
+(use-package s
+  :defer t)
+
+(use-package dash
+  :defer t)
+
+(use-package dash-functional
+  :defer t)
+
+(use-package f
+  :defer t)
+
+(use-package request
+  :commands request)
+
+(use-package ht
+  :defer t)
+
 ;; Elisp utilities
 (defun make-process-sentinel (success err)
   "Makes a process sentinel that calls `success` on success and `err` on error"
@@ -16,4 +38,67 @@
      (when kill-on-err
        (kill-buffer buf)))))
 
+(cl-defun extract-vars-from-env-file (file &key dir)
+  "Extracts an alist of variable name to value from
+   a bash script that exports environment variables."
+  (let ((file (expand-file-name file))
+	(var-re "\\(.+?\\)=\\(.+\\)$")
+	(env '()))
+    (with-temp-buffer
+      (cd (expand-file-name (or dir (file-name-directory file))))
+      (insert (shell-command-to-string (concat "source "
+					       (shell-quote-argument file)
+					       " > /dev/null && env")))
+      (goto-char (point-min))
+      (save-match-data
+	(while (re-search-forward var-re nil t)
+	  (push (cons (match-string 1) (match-string 2)) env))))
+    env))
+
+(defun source-env-file (file)
+  (interactive "fFile: \n")
+  (let ((env (extract-vars-from-env-file file)))
+    (dolist (binding env)
+      (setenv (car binding) (cdr binding)))))
+
+(cl-defmacro with-env-from-file (file &rest body)
+  (declare (indent 1))
+  (let ((env-var (make-symbol "the-env"))
+	(path-var (make-symbol "the-path")))
+    `(let* ((,env-var (extract-vars-from-env-file ,file))
+	    (,path-var (assoc "PATH" ,env-var))
+	    (exec-path
+	     (if ,path-var
+		 (append (split-string (cdr ,path-var) ":") exec-path)
+	       exec-path))
+	    (process-environment
+	     (append
+	      (mapcar
+	       (lambda (elt) (format "%s=%s" (car elt) (cdr elt)))
+	       ,env-var)
+	      process-environment)))
+       ,@body)))
+
+(cl-defun call-with-env-from-file (file callback &key dir)
+  (let* ((env (extract-vars-from-env-file file :dir dir))
+	 (path (assoc "PATH" env))
+	 (exec-path
+	  (if path
+	      (append (split-string (cdr path) ":") exec-path)
+	    exec-path))
+	 (process-environment
+	  (append (mapcar (lambda (elt) (format "%s=%s" (car elt) (cdr elt))) env)
+		  process-environment)))
+    (funcall callback)))
+
+(defmacro with-env (env &rest body)
+  (declare (indent 1))
+  `(let* ((process-environment
+	   (append
+	    (mapcar
+	     (lambda (elt) (format "%s=%s" (car elt) (cdr elt)))
+	     ,env)
+	    process-environment)))
+     ,@body))
+
 (provide 'init-lib)

emacs/.emacs.d/config/init-prodigy.el

@@ -0,0 +1,370 @@
+;; -*- lexical-binding: t; -*-
+
+;; A nice interface for running long-running local programs
+(use-package prodigy
+  :commands (prodigy)
+  :general
+  ('normal prodigy-mode-map "SPC" leader-map)
+  ('normal prodigy-view-mode-map "SPC" leader-map)
+  :config
+  (evil-collection-prodigy-setup)
+  ;; Add ability to associate a file with service logs instead of a buffer
+  (defun prodigy-service-file (service)
+    "Return SERVICE file.
+
+  If SERVICE file exists, use that. If not, find the first SERVICE
+  tag that has a file and return that."
+    (let ((file (prodigy-service-or-first-tag-with service :file)))
+      (if (functionp file)
+	  (prodigy-callback-with-plist file service)
+	file)))
+  (defun prodigy-display-process-file-or-buffer ()
+    (interactive)
+    (when-let (service (prodigy-service-at-pos))
+      (if-let (file (prodigy-service-file service))
+	  (progn
+	    (find-file-literally file)
+	    (prodigy-view-mode)
+	    (auto-revert-tail-mode)
+	    (general-define-key
+	     :states 'normal
+	     :keymaps 'local
+	     "Q" #'kill-this-buffer))
+	(prodigy-switch-to-process-buffer service))))
+  (general-def 'normal prodigy-mode-map "`" #'prodigy-display-process-file-or-buffer)
+  ;; Add ability to inhibit all output processing
+  (defun prodigy-start-service (service &optional callback)
+    "Start process associated with SERVICE unless already started.
+
+  When CALLBACK function is specified, that is called when the
+  process has been started.
+
+  When the process is started, a timer starts and checks every
+  second for `prodigy-start-tryouts' times if the process is live.
+  If the process is not live after `prodigy-start-tryouts' seconds,
+  the process is put in failed status."
+    (declare (indent 1))
+    (unless (prodigy-service-started-p service)
+      (let* ((default-directory
+	       (-if-let (cwd (prodigy-service-cwd service))
+		   (f-full cwd)
+		 default-directory))
+	     (name (plist-get service :name))
+	     (sudo (plist-get service :sudo))
+	     (command (prodigy-service-command service))
+	     (args (prodigy-service-args service))
+	     (exec-path (append (prodigy-service-path service) exec-path))
+	     (env (--map (s-join "=" it) (prodigy-service-env service)))
+	     (process-environment (append env process-environment))
+	     (process nil)
+	     (create-process
+	      (lambda ()
+		(unless process
+		  (setq process (apply (if sudo 'prodigy-start-sudo-process 'start-process)
+				       (append (list name nil  command) args)))))))
+	(-when-let (init (prodigy-service-init service))
+	  (funcall init))
+	(-when-let (init-async (prodigy-service-init-async service))
+	  (let (callbacked)
+	    (funcall
+	     init-async
+	     (lambda ()
+	       (setq callbacked t)
+	       (funcall create-process)))
+	    (with-timeout
+		(prodigy-init-async-timeout
+		 (error "Did not callback async callback within %s seconds"
+			prodigy-init-async-timeout))
+	      (while (not callbacked) (accept-process-output nil 0.005)))))
+	(funcall create-process)
+	(let ((tryout 0))
+	  (prodigy-every 1
+	      (lambda (next)
+		(setq tryout (1+ tryout))
+		(if (process-live-p process)
+		    (when callback (funcall callback))
+		  (if (= tryout prodigy-start-tryouts)
+		      (prodigy-set-status service 'failed)
+		    (funcall next))))))
+	(plist-put service :process process)
+	(when (not (plist-get service :inhibit-process-filter))
+	  (set-process-filter
+	   process
+	   (lambda (_ output)
+	     (run-hook-with-args 'prodigy-process-on-output-hook service output))))
+	(set-process-query-on-exit-flag process nil))))
+  (add-hook 'prodigy-view-mode-hook (lambda () (toggle-truncate-lines 1)))
+
+  ;; Actual service definitions begin here
+  (defun call-with-venv (venv callback)
+    (let ((venv-dir (cond
+                     ((file-exists-p venv) venv)
+                     ((file-exists-p
+                       (substitute-in-file-name
+                        (format "$WORKON_HOME/%s" venv)))
+                      (substitute-in-file-name
+                       (format "$WORKON_HOME/%s" venv)))
+                     (t (error "virtual environment %s does not exist" venv)))))
+      (call-with-env-from-file (format "%s/bin/activate" venv-dir) callback)))
+
+  (defun kill-log-buffers ()
+    (interactive)
+    (kill-matching-buffers "\\.log$" nil t)
+    (message "Killed log buffers"))
+
+  (cl-defun python-service-setup (venv &optional env-file &key env-dir)
+    (lambda (done)
+      (call-with-venv
+       venv
+       (if env-file
+           (lambda ()
+             (call-with-env-from-file env-file done :dir env-dir))
+         done))))
+
+  (defun call-with-lola-env (callback)
+    (let ((process-environment
+           (cons (format "LOLA_ENV=%s"
+                         (completing-read
+                          "Environment: "
+                          '("local" "development" "staging")))
+                 process-environment)))
+      (funcall callback)))
+
+  (prodigy-define-tag
+    :name 'lola)
+
+  (prodigy-define-service
+    :name "lola-server (gunicorn)"
+    :tags '(lola backend)
+    :command "bash"
+    :args (lambda ()
+            (list
+             "-c"
+             "gunicorn -c server/web/gunicorn.conf.py \
+-b 127.0.0.1:7200 bin.start_web:init_and_create_flask_app\\(\\) \
+>> ~/lola/logs/lola-server.log 2>&1"))
+    :file "~/lola/logs/lola-server.log"
+    :inhibit-process-filter t
+    :cwd "~/lola/lola-server"
+    :stop-signal 'int
+    :truncate-output t
+    :init-async (python-service-setup "~/.pyenv/versions/lola-server"
+                                      "~/lola/lola-server/.env"))
+
+  (prodigy-define-service
+    :name "lola-server celery worker"
+    :tags '(lola backend)
+    :command "python"
+    :args '("bin/start_celery_worker.py" "-P" "gevent" "-n" "lola-server")
+    :cwd "~/lola/lola-server"
+    :stop-signal 'int
+    :truncate-output t
+    :init-async (python-service-setup "~/.pyenv/versions/lola-server"
+                                      "~/lola/lola-server/.env"))
+
+  (prodigy-define-service
+    :name "travel-service"
+    :tags '(lola backend)
+    :command "bash"
+    :args (lambda ()
+            (list
+             "-c" "python bin/start_web.py >> ~/lola/logs/travel-svc.log 2>&1"))
+    :cwd "~/lola/lola-travel-service"
+    :file "~/lola/logs/travel-svc.log"
+    :inhibit-process-filter t
+    :stop-signal 'int
+    :truncate-output t
+    :init-async (python-service-setup "~/.pyenv/versions/travel-service"
+                                      "~/lola/lola-travel-service/.env"))
+
+  (prodigy-define-service
+    :name "travel-service celery worker"
+    :tags '(lola backend)
+    :command "bash"
+    :args (lambda ()
+            (list
+             "-c"
+             (concat "python "
+                     "bin/start_celery_workers.py "
+                     "-n " "travel-service "
+                     "-Q "
+                     "default,io_pool,cpu_pool,priority_io_pool,priority_cpu_pool "
+                     ">> ~/lola/logs/travel-svc-celery.log 2>&1")))
+    :file "~/lola/logs/travel-svc-celery.log"
+    :inhibit-process-filter t
+    :cwd "~/lola/lola-travel-service"
+    :stop-signal 'int
+    :truncate-output t
+    :init-async (python-service-setup "~/.pyenv/versions/travel-service"
+                                      "~/lola/lola-travel-service/.env"))
+
+  (prodigy-define-service
+    :name "secrets"
+    :tags '(lola backend)
+    :command "python"
+    :args '("bin/cmdline.py" "www")
+    :cwd "~/lola/secrets"
+    :truncate-output t
+    :stop-signal 'int
+    :init-async (python-service-setup "~/.pyenv/versions/secrets"
+                                      "~/lola/secrets/.env"))
+
+  (prodigy-define-service
+    :name "lola-desktop"
+    :tags '(lola frontend)
+    :command "npm"
+    :args '("start")
+    :cwd "~/lola/lola-desktop"
+    :port 3001
+    :env '(("PORT" "3001"))
+    :stop-signal 'int
+    :init-async #'call-with-lola-env)
+
+  (prodigy-define-service
+    :name "wallet"
+    :tags '(lola frontend)
+    :command "npm"
+    :args '("start")
+    :cwd "~/lola/wallet"
+    :stop-signal 'int
+    :env '(("PORT" "3000"))
+    :init-async #'call-with-lola-env)
+
+  (prodigy-define-service
+    :name "agent-console"
+    :command "npm"
+    :args '("start")
+    :cwd "~/lola/agent-console"
+    :stop-signal 'int
+    :env '(("PORT" "3002"))
+    :init-async (lambda (done)
+                  (call-with-lola-env
+                   (lambda ()
+                     (nvm-use "v10.15.1" done)))))
+
+  (prodigy-define-service
+    :name "luigid"
+    :command "luigid"
+    :cwd "~/lola/data-pipeline"
+    :port 8082
+    :stop-signal 'int
+    :init-async (python-service-setup "data-pipeline"
+                                      "~/lola/data-pipeline/.env"))
+
+  (prodigy-define-service
+    :name "prometheus"
+    :command "prometheus"
+    :args '("--config.file=prometheus.yml")
+    :port 9090
+    :stop-signal 'int
+    :cwd "~/prometheus")
+
+  (prodigy-define-service
+    :name "priceline-service"
+    :tags '(lola backend)
+    :command "~/lola/python_services/priceline/bin/start.sh"
+    :args '("web")
+    :cwd "~/lola/python_services"
+    :stop-signal 'int
+    :init-async (python-service-setup "~/lola/python_services/.venv"
+                                      "~/lola/python_services/priceline/.env"
+                                      :env-dir "~/lola/python_services"))
+
+  (prodigy-define-service
+    :name "priceline-cars-service"
+    :tags '(lola backend)
+    :command "bash"
+    :args '("-c"
+            "priceline_cars/bin/start.sh web >> ~/lola/logs/priceline-cars.log 2>&1")
+    :cwd "~/lola/python_services"
+    :inhibit-process-filter t
+    :file "~/lola/logs/priceline-cars.log"
+    :stop-signal 'int
+    :init-async (python-service-setup "~/lola/python_services/.venv"
+                                      "~/lola/python_services/priceline_cars/.env"
+                                      :env-dir "~/lola/python_services"))
+
+  (prodigy-define-service
+    :name "threev-service"
+    :tags '(lola backend)
+    :command "~/lola/python_services/threev/bin/start.sh"
+    :args '("web")
+    :cwd "~/lola/python_services"
+    :stop-signal 'int
+    :init-async (python-service-setup "~/lola/python_services/.venv"
+                                      "~/lola/python_services/threev/.env"
+                                      :env-dir "~/lola/python_services"))
+
+  (prodigy-define-service
+    :name "amd-flight-service"
+    :tags '(lola backend)
+    :command "~/lola/python_services/amd_flight/bin/start.sh"
+    :args '("web")
+    :cwd "~/lola/python_services"
+    :stop-signal 'int
+    :init-async (python-service-setup "~/lola/python_services/.venv"
+                                      "~/lola/python_services/amd_flight/.env"
+                                      :env-dir "~/lola/python_services"))
+
+  (prodigy-define-service
+    :name "ean-hotels-service"
+    :tags '(lola backend)
+    :command "bash"
+    :args '("-c"
+            "ean_hotels/bin/start.sh web >> ~/lola/logs/ean-hotels.log 2>&1")
+    :cwd "~/lola/python_services"
+    :inhibit-process-filter t
+    :file "~/lola/logs/ean-hotels.log"
+    :stop-signal 'kill
+    :init-async (python-service-setup "~/lola/python_services/.venv"
+                                      "~/lola/python_services/ean_hotels/.env"
+                                      :env-dir "~/lola/python_services"))
+
+  (prodigy-define-service
+    :name "smp-hotels-service"
+    :command "bash"
+    :args '("-c"
+            "smp_hotels/bin/start.sh web >> ~/lola/logs/smp-hotels.log 2>&1")
+    :cwd "~/lola/python_services"
+    :inhibit-process-filter t
+    :file "~/lola/logs/smp-hotels.log"
+    :stop-signal 'kill
+    :init-async (python-service-setup "~/lola/python_services/.venv"
+                                      "~/lola/python_services/smp_hotels/.env"
+                                      :env-dir "~/lola/python_services"))
+
+  (prodigy-define-service
+    :name "email-template-service"
+    :tags '(lola backend)
+    :command "npm"
+    :args '("start")
+    :cwd "~/lola/email-template-service"
+    :env '(("PORT" "7300"))
+    :stop-signal 'int
+    :init-async (lambda (done)
+                  (nvm-use "10.15.1" done)))
+
+  (prodigy-define-service
+    :name "mabl-link-agent"
+    :command "link-agent"
+    :args (lambda ()
+            (list "-a" (password-store-get "mabl-link-agent")
+                  "-n" "jdormit-macbook")))
+
+  (prodigy-define-service
+    :name "xray-daemon"
+    :command "xray_mac"
+    :args '("-o" "-n" "us-east-1"))
+
+  (prodigy-define-service
+    :name "spend-service"
+    :tags '(lola backend)
+    :command "~/lola/python_services/spend/bin/start.sh"
+    :args '("web")
+    :cwd "~/lola/python_services"
+    :stop-signal 'int
+    :init-async (python-service-setup "~/lola/python_services/.venv"
+                                      "~/lola/python_services/spend/.env"
+                                      :env-dir "~/lola/python_services")))
+
+(provide 'init-prodigy)

emacs/.emacs.d/init.el

@@ -77,6 +77,7 @@
 (require 'init-terraform)
 (require 'init-run-command)
 (require 'init-aws)
+(require 'init-prodigy)
 
 ;; Load the custom file
 (when (file-exists-p custom-file)