loop/srv
1
0
Fork 0
Code Issues Pull requests Activity

Implements a REPL for Lisp hackers.

Browse source
This commit is contained in:
Circling Skies 2024-12-14 09:55:33 -03:00
parent e8b57771f5
commit a6a7c766cd
3 changed files with 162 additions and 129 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
Makefile
View file

@ -5,4 +5,5 @@ loop: loop.asd loop.lisp scripts/build-exe.lisp
install: loop
mkdir -p `head -1 conf-home` && \
cp -R loop accounts.lisp groups scripts `head -1 conf-home`
cp -R loop accounts.lisp groups scripts \
`head -1 conf-home`

60
loop.lisp
View file

@ -51,7 +51,8 @@
("DD" ,#'cmd-dd "[d]isplays [d]ata: your state of affairs")
("QUIT" ,#'cmd-quit "politely says good-bye")
("DATE" ,#'cmd-date "displays the current date at this server")
("UNLOCK-ACCOUNT" ,#'cmd-unlock-account "unlocks an account")))
("UNLOCK-ACCOUNT" ,#'cmd-unlock-account "unlocks an account")
("REPL" ,#'cmd-repl "lets you hack away")))
(defun set-up-tables! ()
(labels ((build-commands-assoc (ls)
@ -122,6 +123,10 @@
(make-response :code 400 :data "You must authenticate first.")
(progn ,@body)))
(defun print/finish (&rest args)
(apply #'format (cons t args))
(finish-output))
(defun plural (v suffix)
(if (> v 1) suffix ""))
@ -134,7 +139,8 @@
(defun stderr (&rest args)
(when *debug*
(apply #'out (cons *error-output* args))))
(apply #'out (cons *error-output* args))
(finish-output *error-output*)))
(defun stdout (&rest args)
(apply #'out (list* *standard-output* args)))
@ -180,6 +186,17 @@
(defmacro mac (&rest body)
`(macroexpand-1 ,@body))
(defun repl (r)
(in-package :loop)
(loop
(print/finish "LOOP> ")
(let ((expr (read)))
(if (eq 'quit expr)
(return
(make-response
:code 200 :request r
:data "Okay, no more REPL hacking."))
(println "~a" (eval expr))))))
(defun empty-response () (make-response :code 400 :data "I beg your pardon?"))
(defun prepend-response-with (message r)
(make-response
@ -1120,7 +1137,7 @@
(let ((u (get-account username)))
(when (not u)
(error "I could not find account ~a." username))
(setf (account-pass u) (string->sha256 newpass))
(setf (account-pass u) (string->sha256 (str:upcase newpass)))
(write-accounts!)))
(defun cmd-list-users (r)
@ -1166,6 +1183,9 @@
(if s (universal-to-human s))))))
(defun cmd-dd (r)
(make-response :code 200 :data (format nil "state: ~a" *client*) :request r))
(defun cmd-repl (r)
(with-auth
(repl r)))
(defun notify-group-created (g)
(post-notification
@ -1220,19 +1240,13 @@
:key :list-accounts)
(clingon:make-option
:flag
:description "runs NNTP server reading from stdout"
:short-name #\s
:long-name "server"
:key :server)
(clingon:make-option
:flag
:description "run a REPL on port 4006"
:description "runs a REPL"
:short-name #\r
:long-name "repl"
:key :repl)
(clingon:make-option
:flag
:description "turn on debug logging on stderr"
:description "logging (on stderr)"
:long-name "logging"
:key :logging)))
(defun cli/list-accounts ()
@ -1270,29 +1284,27 @@
(create-index!)
(let ((args (clingon:command-arguments cmd))
(repl (clingon:getopt cmd :repl))
(server (clingon:getopt cmd :server))
(ca (clingon:getopt cmd :create-account))
(change-passwd-account (clingon:getopt cmd :change-passwd))
(list-accounts (clingon:getopt cmd :list-accounts))
(pa (clingon:getopt cmd :change-passwd))
(la (clingon:getopt cmd :list-accounts))
(logging (clingon:getopt cmd :logging)))
(setf *debug* logging)
(when list-accounts
(when la
(cli/list-accounts))
(when ca
(cli/create-account ca args))
(when change-passwd-account
(cli/change-passwd change-passwd-account args))
(when pa
(cli/change-passwd pa args))
(when repl
(stderr "Running a REPL on localhost:4006...~%"))
(when server
(repl))
(when (and (not la) (not ca) (not pa))
(server-start))))
(defun cli/command ()
(clingon:make-command
:name "loop"
:description "An NNTP server for a circle of friends."
:version "0.1"
:authors '("Circling Skies <loop@antartida.xyz>")
:version "0.1" ;; :authors '("Circling Skies <loop@antartida.xyz>")
:license "GPL v3"
:options (cli/options)
:handler #'cli/main))
@ -1300,13 +1312,9 @@
(defun main-loop ()
(let* ((bs (nntp-read-line))
(ln (bytes->string (ucs-2->ascii bs))))
(handler-case
(let ((r (send-response! (dispatch-line ln))))
(when (not (response-quit? r))
(main-loop)))
(SB-SYS:INTERACTIVE-INTERRUPT (c)
(declare (ignore c))
(stderr "^c~%")))))
(main-loop)))))
(defun request-quit? (r) (and r (string= 'quit (request-verb r))))
(defun response-quit? (r) (and r (request-quit? (response-request r))))

220
loop.nw
View file

@ -88,10 +88,10 @@ But \lp\ has nothing to do with e-mail. \Lp\ uses a communication
strategy---called a ``protocol''---that is even older than the web
itself. The web started out around 1989--1990 and the protocol
\lp\ uses begins its history in 1979, the year in which AT\&T released
UNIX V7, which was UUCP-ready. The name UUCP stands for [U]NIX to
[U]NIX [c]o[p]y. It's both a protocol and a set of programs for
copying files between UNIX systems. People begun exchanging messages
by copying files between machines. The idea eventually evolved into a
UNIX V7, which was UUCP-ready, an acronym that stands for for UNIX to
UNIX copy. It's both a protocol and a set of programs for copying
files between UNIX systems. People begun exchanging messages by
copying files between machines. The idea eventually evolved into a
protocol called NNTP---Network News Transfer Protocol---, which is the
protocol used by \lp. (The protocol used to be called Netnews
Transfer Protocol.) Since e-mail was already daily practice of
@ -136,7 +136,7 @@ network news via NNTP.
\label{fg:sylpheed}
\end{figure}
{\bf Principles for a discussion group}. We believe a discussion group
\noindent{\bf Principles for a discussion group}. We believe a discussion group
should be small and grow slowly. By ``slowly'', we mean that each
member comes in through an invitation. This way, the group being
closed by definition, we keep spam out and give members a certain
@ -185,7 +185,7 @@ just want to use the system, you probably should stop right here.
See \href{https://git.antartida.xyz/loop/srv/raw/branch/main/README}{[[README]]}.
\section{The implementation strategy}\label{sec:design}
\section{Implementation strategy}\label{sec:design}
Anything a user sends to the \lp\ is wrapped in a [[request]] and
any command processing must produce a [[response]]:
@ -313,33 +313,28 @@ quit
205 Good-bye.
\end{verbatim}
But keep in mind that \lp\ was not made to talk to users directly.
\Lp\ was made to talk to your NNTP client, programs such as the ones
illustrated by Figures \ref{fg:gnus}--\ref{fg:sylpheed}. That's why
we see these numbers in the responses given by \lp. These numbers are
there to help clients understand how the conversation is going. Each
specific such number is determined by the NNTP protocol. But, despite
the protocol being made for machines to talk to each other, it's
Despite \lp being made to talk to computer programs such as the ones
illustrated in Figures~\ref{fg:gnus}--\ref{fg:sylpheed}, it's
perfectly possible for a user to interact with \lp\ directly using a
keyboard and a command-line tool such as {\tt nc} or {\tt telnet}. In
fact, \lp\ takes advantage of that to be hackable. \Lp\ was written
so that it can talk to NNTP clients---such as Gnus, Sylpheed {\em et
cetera}---but also to users directly. Commands such as
[[CREATE-ACCOUNT]], [[CREATE-GROUP]], [[PASSWD]] are not part of the
NNTP protocol, so users need to know how to use {\tt nc} or {\tt
telnet} to take advantage of all of \lp's capabilities.
fact, \lp\ takes advantage of that to be {\em hackable}. Commands
such as [[CREATE-ACCOUNT]], [[CREATE-GROUP]], [[PASSWD]] are not part
of the NNTP protocol, so users need to know how to use {\tt nc} or
{\tt telnet} to take advantage of all of \lp's capabilities.
\section{The NNTP protocol}
\section{NNTP protocol}
An Internet protocol is usually defined by a document whose tradition
calls RFC---for ``[r]equest [f]or [c]omments''. The NNTP protocol is
defined by RFCs 977, 2980, 3977, 4643 and 5536. RFC 977 was the first
and replaced by 3977. Still, reading RFC 977 is interesting precisely
because it gives us a historical account of the protocol, making it
easier to understanding the evolution of the system. The objective of
RFC 2980 was to implement new ideas to the NNTP protocol---to extend
the protocol. RFC 3977 adopts some of these extensions. RFC 4643
also extends RFC 2980---addressing concerns with authentication.
calls RFC---for ``request for comments''---, an acronym suggested by
Stephen D.~Crocker, who wrote RFC 1 back in 1969, when ARPANET
connected four computers. NNTP is defined by RFCs 977, 2980, 3977,
4643 and 5536. RFC 977 was the first and replaced by 3977. Still,
reading RFC 977 is interesting precisely because it gives us a
historical account of the protocol, making it easier to understand the
evolution of the system. The objective of RFC 2980 was to implement
new ideas to the NNTP protocol---to extend the protocol. RFC 3977
adopts some of these extensions. RFC 4643 also extends RFC
2980---addressing concerns with authentication.
\section{It's a network server}
@ -370,7 +365,36 @@ way, they work together. And \lp\ handles only a text stream, which
is why it's so easy to connect a keyboard to it and interact with it
through the command line as illustrated in Section~\ref{sec:design}.
\section{The description of the package}
\section{REPL for \lp}
There's a REPL to \lp's procedures---always useful.
\begin{verbatim}
%./loop --repl
LOOP> (+ 1 1)
2
LOOP> (list-groups)
(local.control.news local.test)
\end{verbatim}
We implement it by creating a [[repl]] procedure in \lp's package with
an option handled by [[cli/command]].
<<Loop's REPL>>=
(defun repl (r)
(in-package :loop)
(loop
(print/finish "LOOP> ")
(let ((expr (read)))
(if (eq 'quit expr)
(return
(make-response
:code 200 :request r
:data "Okay, no more REPL hacking."))
(println "~a" (eval expr))))))
@ %def repl
\section{Description of the package}
<<Description>>=
An NNTP server for a circle of friends.
@ -382,7 +406,7 @@ An NNTP server for a circle of friends.
These chunks are used in [[loop.asd]].
\section{The representation of a client}
\section{Representation of a client}
How do we represent a client? A client is typically reading a group
and an article; it's has authenticated itself or not yet. So we need
@ -394,7 +418,7 @@ a global structure to annonate the client's state.
@ %def client *client*
\section{The representation of a command}
\section{Representation of a command}
What does a client typically tell \lp? A client typically sends
commands. Commands typically need arguments. Each command is
@ -432,7 +456,8 @@ commands, which is essentially what the user sees when ask for
("DD" ,#'cmd-dd "[d]isplays [d]ata: your state of affairs")
("QUIT" ,#'cmd-quit "politely says good-bye")
("DATE" ,#'cmd-date "displays the current date at this server")
("UNLOCK-ACCOUNT" ,#'cmd-unlock-account "unlocks an account")))
("UNLOCK-ACCOUNT" ,#'cmd-unlock-account "unlocks an account")
("REPL" ,#'cmd-repl "lets you hack away")))
(defun set-up-tables! ()
(labels ((build-commands-assoc (ls)
@ -461,7 +486,7 @@ commands, which is essentially what the user sees when ask for
(or (cdr cmd) (unrecognized-command)))))
@ %def *commands-assoc* set-up-tables! get-command
\section{The representation of requests and responses}
\section{Representation of requests and responses}
Each command is given through a text line written by the user. Let's
call this text line the [[request]]. When \lp\ parses the request, it
@ -538,7 +563,7 @@ else should we handle this?
(write-sequence ls-of-bytes s)))
@ %def my-write
\section{The parsing of requests}
\section{Parsing of requests}
The commands themselves we call {\tt verbs} and everything else the
user types we call {\tt args}. Observe that upper and lower case
@ -557,7 +582,7 @@ letters are equivalent in request verbs.
:args args))))))
@ %def parse-request
\section{The main loop}
\section{Main loop}
Every command consumes a [[request]] and produces a [[response]]. If
any procedure always produces a [[response]], then delivering a
@ -575,13 +600,9 @@ itself---so we can cascade actions based on a user's request.
(defun main-loop ()
(let* ((bs (nntp-read-line))
(ln (bytes->string (ucs-2->ascii bs))))
(handler-case
(let ((r (send-response! (dispatch-line ln))))
(when (not (response-quit? r))
(main-loop)))
(SB-SYS:INTERACTIVE-INTERRUPT (c)
(declare (ignore c))
(stderr "^c~%")))))
(main-loop)))))
(defun request-quit? (r) (and r (string= 'quit (request-verb r))))
(defun response-quit? (r) (and r (request-quit? (response-request r))))
@ -600,13 +621,12 @@ itself---so we can cascade actions based on a user's request.
(make-response :code 200 :data "Welcome! Say ``help'' for a menu.")))
@ %def main main-loop
\section{Command-line parsing}
\section{Parsing of command-line arguments}
We're using the clingon library as per Vincent Dardel suggestion in
``The Common Lisp Cookbook''. (Thanks, Vincent!) We begin with
writing a description of the program and options it understands. XXX:
notice I don't know how to support a two-argument option, so I hacked
a solution away.
``The Common Lisp Cookbook''. We begin with writing a description of
the program and options it understands. XXX: notice I don't know how
to support a two-argument option, so I hacked a solution away.
<<Command-line parsing>>=
(defun cli/options ()
@ -629,19 +649,13 @@ a solution away.
:key :list-accounts)
(clingon:make-option
:flag
:description "runs NNTP server reading from stdout"
:short-name #\s
:long-name "server"
:key :server)
(clingon:make-option
:flag
:description "run a REPL on port 4006"
:description "runs a REPL"
:short-name #\r
:long-name "repl"
:key :repl)
(clingon:make-option
:flag
:description "turn on debug logging on stderr"
:description "logging (on stderr)"
:long-name "logging"
:key :logging)))
@
@ -694,35 +708,33 @@ Now let's write the main procedure in command-line parsing.
(create-index!)
(let ((args (clingon:command-arguments cmd))
(repl (clingon:getopt cmd :repl))
(server (clingon:getopt cmd :server))
(ca (clingon:getopt cmd :create-account))
(change-passwd-account (clingon:getopt cmd :change-passwd))
(list-accounts (clingon:getopt cmd :list-accounts))
(pa (clingon:getopt cmd :change-passwd))
(la (clingon:getopt cmd :list-accounts))
(logging (clingon:getopt cmd :logging)))
(setf *debug* logging)
(when list-accounts
(when la
(cli/list-accounts))
(when ca
(cli/create-account ca args))
(when change-passwd-account
(cli/change-passwd change-passwd-account args))
(when pa
(cli/change-passwd pa args))
(when repl
(stderr "Running a REPL on localhost:4006...~%"))
(when server
(repl))
(when (and (not la) (not ca) (not pa))
(server-start))))
(defun cli/command ()
(clingon:make-command
:name "loop"
:description "<<Description>>"
:version "<<Version>>"
:authors '("Circling Skies <loop@antartida.xyz>")
:version "<<Version>>" ;; :authors '("Circling Skies <loop@antartida.xyz>")
:license "GPL v3"
:options (cli/options)
:handler #'cli/main))
@ %def cli/options cli/command
\section{The request dispatching mechanism}
\section{Request dispatching mechanism}
Dispatching requests means consuming one and invoking the correct
procedure that will process the request. The invoked procedure must
@ -743,7 +755,7 @@ has issued {\tt QUIT}, in which case we terminate [[main-loop]].
(dispatch (parse-request (make-request :said ln))))
@ %def dispatch dispatch-line
\section{The representation and parsing of articles}
\section{Representation and parsing of articles}
An article is made of two parts, the head and the body. We do need to
parse the head, but we never parse the body: we don't want to
@ -872,7 +884,7 @@ using [[lookup]]. I need to seek out all such places and organize. %% TODO
(cdr (assoc key table :test #'string=)))
@ %def extract-mid lookup
\section{The commands}
\section{Commands}
\subsection{{\tt HELP}}
@ -1820,14 +1832,13 @@ noticeable. %%TODO
The NNTP protocol establishes that line termination is done with
\verb|\r\n|, but it's useful to support UNIX line terminations, too,
because we are using the command-line and interact directly with the
server using tools that will not always send \verb|\r\n| as line
because we are using the command-line and interacting directly with
the server using tools that will not always send \verb|\r\n| as line
termination. For example, when someone is typing directly from the
keyboard and insert an empty line, we need the {\tt (and acc ...)}
because sometimes the list [[acc]] comes out empty. But an empty line
because sometimes the list [[acc]] comes out empty, but an empty line
never comes from the NNTP protocol because there's is always a {\tt
CR} before {\tt LF}, but that's not true when someone is using the
keyboard directly.
CR} before {\tt LF}.
<<Command post>>=
(defun nntp-read-line (&optional (s *standard-input*) acc)
@ -1969,7 +1980,21 @@ all or it has been discussed with the community beforehand.
(values nil "must match <<Form of newsgroup names>>"))))
@ %def CREATE-GROUP group-name-conforms?
\section{The publication of news}
\subsection{{\tt REPL}}
Opens a REPL for the pleasure of hacking \lp. The only thing we
require is that the user be authenticated. This means that any user
has total control over \lp. XXX: implement an option
[[--disable-repl]] so that REPL hacking is turned off. (This would
mean your users are not true hackers.)
<<Command repl>>=
(defun cmd-repl (r)
(with-auth
(repl r)))
@
\section{Publication of news}
If you're interested in being notified about what's going on in the
\lp, then subscribe to {\tt local.control.news}. Group creation,
@ -2010,7 +2035,7 @@ invitations {\em et cetera} are published there.
:body (data body)))
@ %def notify-group-created notify-user-created
\section{The algorithm of {\tt split-vector}}
\section{Algorithm of {\tt split-vector}}
How should we describe the algorithm of [[split-vector]]? The
objective is to consume lists of bytes such as
@ -2041,7 +2066,7 @@ searching for the next line.
(+ pos (length delim))))))))
@ %def split-vector
\section{The index article}\label{sec:index}
\section{Article index}\label{sec:index}
Every NNTP server needs to have an index of articles. Each article is
indexed by its message-id. For example, the article
@ -2185,7 +2210,7 @@ doesn't really support symbolic links.
(values grp art))))
@ %def insert-index lookup-index
\section{A procedure to import the index from the file system}
\section{Procedure to import the index from the file system}
%% get group:
%% (first (last (pathname-directory (car (in-groups (directory "**/*"))))))
@ -2224,11 +2249,7 @@ we need to index it.
Here's a program to build the index from a UNIX shell.
<<build-index-from-fs.lisp>>=
#-quicklisp
(let ((quicklisp-init (merge-pathnames "quicklisp/setup.lisp"
(user-homedir-pathname))))
(when (probe-file quicklisp-init)
(load quicklisp-init)))
<<Quicklisp loading preamble>>
(ql:quickload :loop :silent t)
(in-package #:loop)
(connect-index! "message-id.db")
@ -2266,11 +2287,7 @@ The program [[cron-remove-inactive-users.lisp]] can be executed every
day at midnight, say.
<<cron-remove-inactive-users.lisp>>=
#-quicklisp
(let ((quicklisp-init (merge-pathnames "quicklisp/setup.lisp"
(user-homedir-pathname))))
(when (probe-file quicklisp-init)
(load quicklisp-init)))
<<Quicklisp loading preamble>>
(ql:quickload :loop :silent t)
(in-package #:loop)
(read-accounts!)
@ -2358,11 +2375,7 @@ New system administrators of \Lp\ will never need to run this.
Here's a program to run the migration in a UNIX shell.
<<migrate-add-creation-date.lisp>>=
#-quicklisp
(let ((quicklisp-init (merge-pathnames "quicklisp/setup.lisp"
(user-homedir-pathname))))
(when (probe-file quicklisp-init)
(load quicklisp-init)))
<<Quicklisp loading preamble>>
(ql:quickload :loop :silent t)
(in-package #:loop)
(migrate-add-creation-and-post-date!)
@ -2429,7 +2442,7 @@ Now we write the procedures that discover what accounts are inactive.
(user-inactive? (account-username u)))))
@ %def list-inactive-users
\section{A special-purpose language to ease writing}\label{sec:dsl}
\section{Macros to ease writing}\label{sec:dsl}
These macros make up a tiny language to ease the writing of \lp. For
example, when we need to access the group database, we use
@ -2494,6 +2507,10 @@ stands for ``Universal Character Set'' and I speculate the number 2
means 2 bytes. So our conversion is just removing the first byte.
<<Little procedures>>=
(defun print/finish (&rest args)
(apply #'format (cons t args))
(finish-output))
(defun plural (v suffix)
(if (> v 1) suffix ""))
@ -2506,7 +2523,8 @@ means 2 bytes. So our conversion is just removing the first byte.
(defun stderr (&rest args)
(when *debug*
(apply #'out (cons *error-output* args))))
(apply #'out (cons *error-output* args))
(finish-output *error-output*)))
(defun stdout (&rest args)
(apply #'out (list* *standard-output* args)))
@ -2587,11 +2605,7 @@ the code.
Just say {\tt make loop} to your shell.
<<build-exe.lisp>>=
#-quicklisp
(let ((quicklisp-init (merge-pathnames "quicklisp/setup.lisp"
(user-homedir-pathname))))
(when (probe-file quicklisp-init)
(load quicklisp-init)))
<<Quicklisp loading preamble>>
(ql:quickload :loop)
(sb-ext:save-lisp-and-die #P"loop"
:toplevel #'loop:main
@ -2599,6 +2613,14 @@ Just say {\tt make loop} to your shell.
:save-runtime-options t)
@
<<Quicklisp loading preamble>>=
#-quicklisp
(let ((quicklisp-init (merge-pathnames "quicklisp/setup.lisp"
(user-homedir-pathname))))
(when (probe-file quicklisp-init)
(load quicklisp-init)))
@ %def quicklisp
\section{How to get a log of \lp's communication}
If you invoke \lxxp\ with option [[--logging]], you get logging on
@ -2636,7 +2658,7 @@ specify in {\tt syslog.conf} the selector {\tt user.notice} and
specify a log file such as {\tt /var/log/loop.log}. See {\tt
syslog.conf(5)} and {\tt newsyslog(1)} for more information.
\section{The package {\tt loop.lisp} as the compiler needs it}
\section{Package {\tt loop.lisp} as the compiler needs it}
We now put together all source code chunks in the order the compiler
needs to read it. One thing to keep in mind here is---I wonder if
@ -2673,6 +2695,7 @@ something to think about.
<<Table of commands>>
<<Macros>>
<<Little procedures>>
<<Loop's REPL>>
<<Procedures for requests and responses>>
<<Essential operations relative to the index>>
<<How to create and connect to the index>>
@ -2699,6 +2722,7 @@ something to think about.
<<Command passwd>>
<<Command users>>
<<Command dd>>
<<Command repl>>
<<Broadcasting>>