From 11080e243ad4111d84f67398bd453dd2df37ea4b Mon Sep 17 00:00:00 2001 From: Jeremy Dormitzer Date: Sun, 21 Feb 2021 08:20:46 -0500 Subject: [PATCH] Configure prodigy --- emacs/.emacs.d/config/init-lib.el | 85 ++++++ emacs/.emacs.d/config/init-prodigy.el | 370 ++++++++++++++++++++++++++ emacs/.emacs.d/init.el | 1 + 3 files changed, 456 insertions(+) create mode 100644 emacs/.emacs.d/config/init-prodigy.el diff --git a/emacs/.emacs.d/config/init-lib.el b/emacs/.emacs.d/config/init-lib.el index 9da8d0d..6b1f7a0 100644 --- a/emacs/.emacs.d/config/init-lib.el +++ b/emacs/.emacs.d/config/init-lib.el @@ -1,4 +1,26 @@ ;; -*- lexical-binding: t; -*- +;; Helper packages +(use-package deferred + :defer t) + +(use-package s + :defer t) + +(use-package dash + :defer t) + +(use-package dash-functional + :defer t) + +(use-package f + :defer t) + +(use-package request + :commands request) + +(use-package ht + :defer t) + ;; Elisp utilities (defun make-process-sentinel (success err) "Makes a process sentinel that calls `success` on success and `err` on error" @@ -16,4 +38,67 @@ (when kill-on-err (kill-buffer buf))))) +(cl-defun extract-vars-from-env-file (file &key dir) + "Extracts an alist of variable name to value from + a bash script that exports environment variables." + (let ((file (expand-file-name file)) + (var-re "\\(.+?\\)=\\(.+\\)$") + (env '())) + (with-temp-buffer + (cd (expand-file-name (or dir (file-name-directory file)))) + (insert (shell-command-to-string (concat "source " + (shell-quote-argument file) + " > /dev/null && env"))) + (goto-char (point-min)) + (save-match-data + (while (re-search-forward var-re nil t) + (push (cons (match-string 1) (match-string 2)) env)))) + env)) + +(defun source-env-file (file) + (interactive "fFile: \n") + (let ((env (extract-vars-from-env-file file))) + (dolist (binding env) + (setenv (car binding) (cdr binding))))) + +(cl-defmacro with-env-from-file (file &rest body) + (declare (indent 1)) + (let ((env-var (make-symbol "the-env")) + (path-var (make-symbol "the-path"))) + `(let* ((,env-var (extract-vars-from-env-file ,file)) + (,path-var (assoc "PATH" ,env-var)) + (exec-path + (if ,path-var + (append (split-string (cdr ,path-var) ":") exec-path) + exec-path)) + (process-environment + (append + (mapcar + (lambda (elt) (format "%s=%s" (car elt) (cdr elt))) + ,env-var) + process-environment))) + ,@body))) + +(cl-defun call-with-env-from-file (file callback &key dir) + (let* ((env (extract-vars-from-env-file file :dir dir)) + (path (assoc "PATH" env)) + (exec-path + (if path + (append (split-string (cdr path) ":") exec-path) + exec-path)) + (process-environment + (append (mapcar (lambda (elt) (format "%s=%s" (car elt) (cdr elt))) env) + process-environment))) + (funcall callback))) + +(defmacro with-env (env &rest body) + (declare (indent 1)) + `(let* ((process-environment + (append + (mapcar + (lambda (elt) (format "%s=%s" (car elt) (cdr elt))) + ,env) + process-environment))) + ,@body)) + (provide 'init-lib) diff --git a/emacs/.emacs.d/config/init-prodigy.el b/emacs/.emacs.d/config/init-prodigy.el new file mode 100644 index 0000000..25defbf --- /dev/null +++ b/emacs/.emacs.d/config/init-prodigy.el @@ -0,0 +1,370 @@ +;; -*- lexical-binding: t; -*- + +;; A nice interface for running long-running local programs +(use-package prodigy + :commands (prodigy) + :general + ('normal prodigy-mode-map "SPC" leader-map) + ('normal prodigy-view-mode-map "SPC" leader-map) + :config + (evil-collection-prodigy-setup) + ;; Add ability to associate a file with service logs instead of a buffer + (defun prodigy-service-file (service) + "Return SERVICE file. + + If SERVICE file exists, use that. If not, find the first SERVICE + tag that has a file and return that." + (let ((file (prodigy-service-or-first-tag-with service :file))) + (if (functionp file) + (prodigy-callback-with-plist file service) + file))) + (defun prodigy-display-process-file-or-buffer () + (interactive) + (when-let (service (prodigy-service-at-pos)) + (if-let (file (prodigy-service-file service)) + (progn + (find-file-literally file) + (prodigy-view-mode) + (auto-revert-tail-mode) + (general-define-key + :states 'normal + :keymaps 'local + "Q" #'kill-this-buffer)) + (prodigy-switch-to-process-buffer service)))) + (general-def 'normal prodigy-mode-map "`" #'prodigy-display-process-file-or-buffer) + ;; Add ability to inhibit all output processing + (defun prodigy-start-service (service &optional callback) + "Start process associated with SERVICE unless already started. + + When CALLBACK function is specified, that is called when the + process has been started. + + When the process is started, a timer starts and checks every + second for `prodigy-start-tryouts' times if the process is live. + If the process is not live after `prodigy-start-tryouts' seconds, + the process is put in failed status." + (declare (indent 1)) + (unless (prodigy-service-started-p service) + (let* ((default-directory + (-if-let (cwd (prodigy-service-cwd service)) + (f-full cwd) + default-directory)) + (name (plist-get service :name)) + (sudo (plist-get service :sudo)) + (command (prodigy-service-command service)) + (args (prodigy-service-args service)) + (exec-path (append (prodigy-service-path service) exec-path)) + (env (--map (s-join "=" it) (prodigy-service-env service))) + (process-environment (append env process-environment)) + (process nil) + (create-process + (lambda () + (unless process + (setq process (apply (if sudo 'prodigy-start-sudo-process 'start-process) + (append (list name nil command) args))))))) + (-when-let (init (prodigy-service-init service)) + (funcall init)) + (-when-let (init-async (prodigy-service-init-async service)) + (let (callbacked) + (funcall + init-async + (lambda () + (setq callbacked t) + (funcall create-process))) + (with-timeout + (prodigy-init-async-timeout + (error "Did not callback async callback within %s seconds" + prodigy-init-async-timeout)) + (while (not callbacked) (accept-process-output nil 0.005))))) + (funcall create-process) + (let ((tryout 0)) + (prodigy-every 1 + (lambda (next) + (setq tryout (1+ tryout)) + (if (process-live-p process) + (when callback (funcall callback)) + (if (= tryout prodigy-start-tryouts) + (prodigy-set-status service 'failed) + (funcall next)))))) + (plist-put service :process process) + (when (not (plist-get service :inhibit-process-filter)) + (set-process-filter + process + (lambda (_ output) + (run-hook-with-args 'prodigy-process-on-output-hook service output)))) + (set-process-query-on-exit-flag process nil)))) + (add-hook 'prodigy-view-mode-hook (lambda () (toggle-truncate-lines 1))) + + ;; Actual service definitions begin here + (defun call-with-venv (venv callback) + (let ((venv-dir (cond + ((file-exists-p venv) venv) + ((file-exists-p + (substitute-in-file-name + (format "$WORKON_HOME/%s" venv))) + (substitute-in-file-name + (format "$WORKON_HOME/%s" venv))) + (t (error "virtual environment %s does not exist" venv))))) + (call-with-env-from-file (format "%s/bin/activate" venv-dir) callback))) + + (defun kill-log-buffers () + (interactive) + (kill-matching-buffers "\\.log$" nil t) + (message "Killed log buffers")) + + (cl-defun python-service-setup (venv &optional env-file &key env-dir) + (lambda (done) + (call-with-venv + venv + (if env-file + (lambda () + (call-with-env-from-file env-file done :dir env-dir)) + done)))) + + (defun call-with-lola-env (callback) + (let ((process-environment + (cons (format "LOLA_ENV=%s" + (completing-read + "Environment: " + '("local" "development" "staging"))) + process-environment))) + (funcall callback))) + + (prodigy-define-tag + :name 'lola) + + (prodigy-define-service + :name "lola-server (gunicorn)" + :tags '(lola backend) + :command "bash" + :args (lambda () + (list + "-c" + "gunicorn -c server/web/gunicorn.conf.py \ +-b 127.0.0.1:7200 bin.start_web:init_and_create_flask_app\\(\\) \ +>> ~/lola/logs/lola-server.log 2>&1")) + :file "~/lola/logs/lola-server.log" + :inhibit-process-filter t + :cwd "~/lola/lola-server" + :stop-signal 'int + :truncate-output t + :init-async (python-service-setup "~/.pyenv/versions/lola-server" + "~/lola/lola-server/.env")) + + (prodigy-define-service + :name "lola-server celery worker" + :tags '(lola backend) + :command "python" + :args '("bin/start_celery_worker.py" "-P" "gevent" "-n" "lola-server") + :cwd "~/lola/lola-server" + :stop-signal 'int + :truncate-output t + :init-async (python-service-setup "~/.pyenv/versions/lola-server" + "~/lola/lola-server/.env")) + + (prodigy-define-service + :name "travel-service" + :tags '(lola backend) + :command "bash" + :args (lambda () + (list + "-c" "python bin/start_web.py >> ~/lola/logs/travel-svc.log 2>&1")) + :cwd "~/lola/lola-travel-service" + :file "~/lola/logs/travel-svc.log" + :inhibit-process-filter t + :stop-signal 'int + :truncate-output t + :init-async (python-service-setup "~/.pyenv/versions/travel-service" + "~/lola/lola-travel-service/.env")) + + (prodigy-define-service + :name "travel-service celery worker" + :tags '(lola backend) + :command "bash" + :args (lambda () + (list + "-c" + (concat "python " + "bin/start_celery_workers.py " + "-n " "travel-service " + "-Q " + "default,io_pool,cpu_pool,priority_io_pool,priority_cpu_pool " + ">> ~/lola/logs/travel-svc-celery.log 2>&1"))) + :file "~/lola/logs/travel-svc-celery.log" + :inhibit-process-filter t + :cwd "~/lola/lola-travel-service" + :stop-signal 'int + :truncate-output t + :init-async (python-service-setup "~/.pyenv/versions/travel-service" + "~/lola/lola-travel-service/.env")) + + (prodigy-define-service + :name "secrets" + :tags '(lola backend) + :command "python" + :args '("bin/cmdline.py" "www") + :cwd "~/lola/secrets" + :truncate-output t + :stop-signal 'int + :init-async (python-service-setup "~/.pyenv/versions/secrets" + "~/lola/secrets/.env")) + + (prodigy-define-service + :name "lola-desktop" + :tags '(lola frontend) + :command "npm" + :args '("start") + :cwd "~/lola/lola-desktop" + :port 3001 + :env '(("PORT" "3001")) + :stop-signal 'int + :init-async #'call-with-lola-env) + + (prodigy-define-service + :name "wallet" + :tags '(lola frontend) + :command "npm" + :args '("start") + :cwd "~/lola/wallet" + :stop-signal 'int + :env '(("PORT" "3000")) + :init-async #'call-with-lola-env) + + (prodigy-define-service + :name "agent-console" + :command "npm" + :args '("start") + :cwd "~/lola/agent-console" + :stop-signal 'int + :env '(("PORT" "3002")) + :init-async (lambda (done) + (call-with-lola-env + (lambda () + (nvm-use "v10.15.1" done))))) + + (prodigy-define-service + :name "luigid" + :command "luigid" + :cwd "~/lola/data-pipeline" + :port 8082 + :stop-signal 'int + :init-async (python-service-setup "data-pipeline" + "~/lola/data-pipeline/.env")) + + (prodigy-define-service + :name "prometheus" + :command "prometheus" + :args '("--config.file=prometheus.yml") + :port 9090 + :stop-signal 'int + :cwd "~/prometheus") + + (prodigy-define-service + :name "priceline-service" + :tags '(lola backend) + :command "~/lola/python_services/priceline/bin/start.sh" + :args '("web") + :cwd "~/lola/python_services" + :stop-signal 'int + :init-async (python-service-setup "~/lola/python_services/.venv" + "~/lola/python_services/priceline/.env" + :env-dir "~/lola/python_services")) + + (prodigy-define-service + :name "priceline-cars-service" + :tags '(lola backend) + :command "bash" + :args '("-c" + "priceline_cars/bin/start.sh web >> ~/lola/logs/priceline-cars.log 2>&1") + :cwd "~/lola/python_services" + :inhibit-process-filter t + :file "~/lola/logs/priceline-cars.log" + :stop-signal 'int + :init-async (python-service-setup "~/lola/python_services/.venv" + "~/lola/python_services/priceline_cars/.env" + :env-dir "~/lola/python_services")) + + (prodigy-define-service + :name "threev-service" + :tags '(lola backend) + :command "~/lola/python_services/threev/bin/start.sh" + :args '("web") + :cwd "~/lola/python_services" + :stop-signal 'int + :init-async (python-service-setup "~/lola/python_services/.venv" + "~/lola/python_services/threev/.env" + :env-dir "~/lola/python_services")) + + (prodigy-define-service + :name "amd-flight-service" + :tags '(lola backend) + :command "~/lola/python_services/amd_flight/bin/start.sh" + :args '("web") + :cwd "~/lola/python_services" + :stop-signal 'int + :init-async (python-service-setup "~/lola/python_services/.venv" + "~/lola/python_services/amd_flight/.env" + :env-dir "~/lola/python_services")) + + (prodigy-define-service + :name "ean-hotels-service" + :tags '(lola backend) + :command "bash" + :args '("-c" + "ean_hotels/bin/start.sh web >> ~/lola/logs/ean-hotels.log 2>&1") + :cwd "~/lola/python_services" + :inhibit-process-filter t + :file "~/lola/logs/ean-hotels.log" + :stop-signal 'kill + :init-async (python-service-setup "~/lola/python_services/.venv" + "~/lola/python_services/ean_hotels/.env" + :env-dir "~/lola/python_services")) + + (prodigy-define-service + :name "smp-hotels-service" + :command "bash" + :args '("-c" + "smp_hotels/bin/start.sh web >> ~/lola/logs/smp-hotels.log 2>&1") + :cwd "~/lola/python_services" + :inhibit-process-filter t + :file "~/lola/logs/smp-hotels.log" + :stop-signal 'kill + :init-async (python-service-setup "~/lola/python_services/.venv" + "~/lola/python_services/smp_hotels/.env" + :env-dir "~/lola/python_services")) + + (prodigy-define-service + :name "email-template-service" + :tags '(lola backend) + :command "npm" + :args '("start") + :cwd "~/lola/email-template-service" + :env '(("PORT" "7300")) + :stop-signal 'int + :init-async (lambda (done) + (nvm-use "10.15.1" done))) + + (prodigy-define-service + :name "mabl-link-agent" + :command "link-agent" + :args (lambda () + (list "-a" (password-store-get "mabl-link-agent") + "-n" "jdormit-macbook"))) + + (prodigy-define-service + :name "xray-daemon" + :command "xray_mac" + :args '("-o" "-n" "us-east-1")) + + (prodigy-define-service + :name "spend-service" + :tags '(lola backend) + :command "~/lola/python_services/spend/bin/start.sh" + :args '("web") + :cwd "~/lola/python_services" + :stop-signal 'int + :init-async (python-service-setup "~/lola/python_services/.venv" + "~/lola/python_services/spend/.env" + :env-dir "~/lola/python_services"))) + +(provide 'init-prodigy) diff --git a/emacs/.emacs.d/init.el b/emacs/.emacs.d/init.el index 451d21a..92c4145 100644 --- a/emacs/.emacs.d/init.el +++ b/emacs/.emacs.d/init.el @@ -77,6 +77,7 @@ (require 'init-terraform) (require 'init-run-command) (require 'init-aws) +(require 'init-prodigy) ;; Load the custom file (when (file-exists-p custom-file)