jdormit/structlog-el
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Basic functionality: querying + edit current query, some keybindings

Browse Source
This commit is contained in:
Jeremy Dormitzer 2020-06-03 13:26:39 -04:00
parent db5e819acc
commit e86983d522
1 changed files with 194 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

212
structlog-mode.el
View File

@ -1,29 +1,202 @@
;;; structlog-mode.el --- Query structured logs -*- lexical-binding: t; -*-
;; Copyright (C) 2020 Jeremy Dormitzer
;; Author: Jeremy Dormitzer <jdormit@Jeremys-MBP>
;; Version: 0.1.0
;; Package-Requires ((dash "2.17.0") (s "1.12.0"))
;; Keywords: data
;; This program is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation, either version 3 of the License, or
;; (at your option) any later version.
;; This program is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;; GNU General Public License for more details.
;; You should have received a copy of the GNU General Public License
;; along with this program. If not, see <https://www.gnu.org/licenses/>.
;;; Commentary:
;; structlog-mode provides an interface to query structured logs
;; stored in a database. Right now, it only handles logs stored in the
;; format dictated by the pgjson Fluentd plugin.
;;; Code:
(require 'cl-lib)
(require 's)
(require 'dash)
(defvar structlog-fields nil
"Currently selected structlog fields")
(defvar structlog-logs nil
"Current structlog log lines, formatted as plists")
(defvar structlog-db-username nil
"PostgreSQL username to use for structlog-mode")
(defvar structlog-db-database nil
"PostgreSQL database to use for structlog-mode")
(defvar structlog-db-password nil
"PostgresSQL password to use for structlog-mode")
(defvar structlog-db-table "fluentd"
"PostgresSQL table to use for structlog-mode")
(defvar structlog-db-record-field "record"
"PostgreSQL column used to store the log line records")
(defvar structlog-psql-path (executable-find "psql")
"Path to the psql executable")
(defvar structlog-current-query nil
"Query for the current structlog display")
(defvar structlog-field-default-width 40
"Default width for log lines")
(defvar structlog-field-widths '((time . 25)
(event . 100))
"Widths for specific fields, overrides
`structlog-field-default-width'")
(defvar structlog-default-query
":select [time event] :limit 100"
"Default query for `structlog' command")
(defun structlog--query-db (query)
"Runs `query' against the database, returning a list of lists"
(dolist (var '(structlog-db-username
structlog-db-database))
(when (or (not (boundp var))
(not (symbol-value var)))
(error "%s must be set" var)))
(let ((raw (shell-command-to-string
(concat (when structlog-db-password
(format "PGPASSWORD=%s " structlog-db-password))
structlog-psql-path " "
"-U " structlog-db-username " "
"-d " structlog-db-database " "
"-F '<>STRUCTLOG_SEP<>' "
"-t -A -P pager "
"-c " (shell-quote-argument query)))))
(->> raw
(s-lines)
(--filter (not (string-empty-p it)))
(mapcar (apply-partially #'s-split "<>STRUCTLOG_SEP<>")))))
(defun structlog--parse-json (raw)
"Parses the string `raw' as JSON and returns an alist"
(if (fboundp 'json-parse-string)
(json-parse-string raw
:object-type 'alist
:null-object nil
:false-object nil)
(require 'json)
(json-read-from-string raw)))
(defun structlog--get-query-fields (query)
"Returns the fields selected by `query'"
(plist-get query :select))
(defun structlog--get-query-sql (query)
"Returns the SQL query to run for `query'"
(let ((base (format "SELECT %s FROM %s"
structlog-db-record-field
structlog-db-table))
(limit (plist-get query :limit)))
(concat base
(when limit
(format " LIMIT %s" limit)))))
(defun structlog--make-list-entries ()
"Makes the tabulated-list-mode entries list for structlog"
(cl-map 'list
(lambda (log-plist)
(list log-plist
(cl-map 'vector
(lambda (field)
(plist-get log-plist field))
structlog-fields)))
structlog-logs))
(let* ((fields (structlog--get-query-fields structlog-current-query))
(sql (structlog--get-query-sql structlog-current-query))
(raw (structlog--query-db sql))
(records (->> raw
(-map #'car)
(-map #'structlog--parse-json))))
(setq structlog-fields fields)
(setq structlog-logs records)
(cl-map 'list
(lambda (log-alist)
(list log-alist
(cl-map 'vector
(lambda (field)
(alist-get field log-alist))
structlog-fields)))
structlog-logs)))
(defun structlog--query->str (query)
"Converts `query' to a string representation by unwrapping it"
(->> (prin1-to-string query)
(s-chop-prefix "(")
(s-chop-suffix ")")))
(defun structlog--str->query (str)
"Converts `str' into a query list"
(->> str
(s-prepend "(")
(s-append ")")
(read-from-string)
(car)))
(defun structlog--fields->format ()
(cl-map 'vector
(lambda (field)
(list (symbol-name field)
(or (alist-get field structlog-field-widths)
structlog-field-default-width)
t))
structlog-fields))
(defun structlog--revert (&rest _)
(setq structlog-fields (structlog--get-query-fields
structlog-current-query)
tabulated-list-format (structlog--fields->format))
(tabulated-list-revert)
(tabulated-list-init-header))
(defun structlog-query (query)
(interactive "sQuery: ")
"Runs `query' and re-renders the table"
(setq structlog-current-query (structlog--str->query query))
(structlog--revert))
(defun structlog-edit-query ()
(interactive)
(let* ((query-str (structlog--query->str structlog-current-query))
(new-query (read-string "Query: " query-str)))
(structlog-query new-query)))
(defvar structlog-mode-map
(let ((map (make-sparse-keymap)))
(set-keymap-parent map tabulated-list-mode-map)
(define-key map "m" #'structlog-query)
(define-key map "M" #'structlog-edit-query)
map)
"Local keymap for `structlog-mode' buffers")
(when (fboundp 'evil-define-key)
(evil-define-key 'normal structlog-mode-map
"m" #'structlog-query
"M" #'structlog-edit-query
"S" #'tabulated-list-sort
"{" #'tabulated-list-narrow-current-column
"}" #'tabulated-list-widen-current-column))
(define-derived-mode structlog-mode tabulated-list-mode "structlog"
"Major mode to query structured log lines"
(setq tabulated-list-format
(cl-map 'vector
(lambda (field)
(list (symbol-name field) 20 t))
structlog-fields)
(setq tabulated-list-format (structlog--fields->format)
tabulated-list-entries #'structlog--make-list-entries)
(setq-local revert-buffer-function #'structlog--revert)
(tabulated-list-init-header))
;;;###autoload
@ -32,11 +205,14 @@
(when (get-buffer "*structlog*")
(kill-buffer "*structlog*"))
(with-current-buffer (get-buffer-create "*structlog*")
;; TODO do something real here
(setq structlog-fields '(time event)
structlog-logs '((time "2020-06-02T16:55:00" event "Got a baz")
(time "2020-06-02T17:00:00" event "Got a bar")
(time "2020-06-02T17:04:00" event "Got a foo")))
(setq structlog-current-query (structlog--str->query
(read-string "Query: "
structlog-default-query))
structlog-fields (structlog--get-query-fields
structlog-current-query))
(structlog-mode)
(tabulated-list-print))
(switch-to-buffer "*structlog*"))
(provide 'structlog-mode)
;;; structlog-mode.el ends here