Browse Source

Simplified persistence

tags/v0.17.0
Alex Williams 1 year ago
parent
commit
6d043ab5f5
Signed by: aw GPG Key ID: 19EE4AAA361A7E2C
  1. 13
      CHANGELOG.md
  2. 113
      libkv.l
  3. 2
      libkvclient.l
  4. 2
      module.l
  5. 9
      test.l
  6. 14
      test/test_kv.l

13
CHANGELOG.md

@ -1,5 +1,18 @@
# Changelog
## 0.17.0 (2020-08-20)
### Bug fixes
* Catch all IO errors when saving the database to disk
### Misc changes
* Simplified the database save process (persistence)
* Foreground database saving has been removed, it will _always_ happen in the background
* The AOF is opened/closed for every write as opposed to being open forever
* The `SAVE` command is now exactly the same as `BGSAVE`
## 0.16.0 (2020-08-17)
### New features

113
libkv.l

@ -54,7 +54,7 @@
(case (uppc (car Request))
["APPEND" (kv-cmd-append Key Value) ]
["BGSAVE" (kv-bgsave-db *Aof_desc) ]
["BGSAVE" (kv-bgsave-db) ]
["CLIENT" (kv-cmd-client Child (cadr Request) ]
["CONVERT" (kv-cmd-convert) ]
["DEL" (kv-cmd-del Key (cdadr Request) ]
@ -88,7 +88,7 @@
["RPOP" (kv-cmd-rpop Key) ]
["RPOPLPUSH" (kv-cmd-rpoplpush Key Value) ]
["RPUSH" (kv-cmd-rpush Key (cdadr Request) ]
["SAVE" (kv-save-db) ]
["SAVE" (kv-bgsave-db) ] # same as BGSAVE
["SET" (kv-cmd-set Key Value) ]
["STRLEN" (kv-cmd-strlen Key) ]
[T "Error: Unknown command" ] ]
@ -386,37 +386,6 @@
# PERSISTENCE
###
# Rewrite the AOF with new entries if they were added
[de kv-rewrite-aof ()
(ctl *KV_aof_lock
(one *KV/%stats%/aof_rewrite_in_progress)
(when (info *KV_aof_tmp)
(kv-output "====== Rewriting AOF ======")
(out (pack "+" *KV_aof_tmp) (in *KV_aof (echo))) # Append the current AOF into the temporary AOF
(out *KV_aof (in *KV_aof_tmp (echo))) # Copy the temporary AOF into the current AOF
(call 'rm "-f" *KV_aof_tmp)
(kv-output "====== AOF saved ======") )
(zero *KV/%stats%/aof_rewrite_in_progress) ]
[de kv-remove-aof (Bg)
(unless Bg (out *KV_aof (rewind)))
(call 'rm "-f" *KV_aof_tmp) ]
# Write the new DB to disk
[de kv-write-db ()
(kv-stat "rdb_last_cow_size" (car (info *KV_db_tmp)))
(and
(if (info *KV_db)
(call 'cp *KV_db (pack *KV_db ".old"))
T )
(or (kv-output "====== Writing DB ======") T)
(call 'mv *KV_db_tmp *KV_db) )
(or (kv-output "====== DB saved ======") T) ]
# Write data to the DB, then write the AOF (truncate or wipe)
[de kv-write-data (Bg)
(and (info *KV_db_tmp) (kv-write-db) (kv-remove-aof Bg) ]
# Write the data in binary PLIO (pr) or plaintext (println) format
[de kv-save-data (Key)
(let Result (kv-value Key)
@ -430,29 +399,37 @@
(out *KV_db_tmp
(mapcar kv-save-data (kv-cmd-get "%stats%/keys") ]
# Perform some maintenance tasks when save ends
[de kv-save-cleanup ()
(call 'rm "-f" *KV_aof_lock *KV_db_lock) ]
# Obtain a UNIX timestamp
[de kv-timestamp (Ns)
(in (list 'date (if Ns "+%s.%N" "+%s")) (line T) ]
# Save the entire DB keyspace to a file
[de kv-save-db (Bg)
(if (kv-locked?)
(kv-rewrite-aof) # restore the AOF if the DB is locked
(out *KV_db_lock (prinl *Pid))
(kv-output "[dbwriter]=" *Pid " Saving the DB to " *KV_db)
(kv-stat "rdb_last_save_time" (kv-timestamp))
# Rewrite the AOF with new entries if they were added
[de kv-rewrite-aof ()
(ctl *KV_aof_lock
(when *Msg (out 2 (prinl "^J======^JERROR: " *Msg "^J======^J"))) # Print error message to STDERR
(when (info *KV_aof_tmp)
(kv-output "====== Rewriting AOF ======")
(out (pack "+" *KV_aof_tmp) (in *KV_aof (echo))) # Append the current AOF into the temporary AOF
(out *KV_aof (in *KV_aof_tmp (echo))) # Copy the temporary AOF into the current AOF
(call 'rm "-f" *KV_aof_tmp)
(kv-output "====== AOF saved ======") ) )
(finally
(kv-save-cleanup)
(kv-save-db-keys)
(kv-write-data Bg)
(unless *PPid (bye))
(call 'rm "-f" *KV_aof_lock *KV_db_lock) ] # Remove temporary AOF and DB locks
(kv-stat "rdb_last_bgsave_status" "OK") ]
# Save the entire DB keyspace to a file
[de kv-save-db ()
(kv-output "[dbwriter]=" *Pid " Saving the DB to " *KV_db)
(catch '(NIL)
(finally
(kv-rewrite-aof)
(kv-output "====== Writing DB ======")
(kv-save-db-keys)
(call 'cp *KV_db (pack *KV_db ".old")) # Backup the DB file
(call 'mv *KV_db_tmp *KV_db) # Write the new DB to disk
(call 'rm "-f" *KV_aof_tmp) # Remove the temporary AOF file
(kv-output "====== DB saved ======")
(bye) ]
# Check if the DB is locked for writing, and return the error message
[de kv-locked? ()
@ -461,13 +438,14 @@
(kv-stat "rdb_last_bgsave_status" "Error: DB is locked for writing") ]
# Save the entire DB keyspace to a file in the background (fork)
[de kv-bgsave-db (Aof)
[de kv-bgsave-db ()
(if (kv-locked?)
@
(out *KV_db_lock (prinl *Pid))
(kv-stat "rdb_last_save_time" (kv-timestamp))
(call 'cp *KV_aof *KV_aof_tmp) # make a copy of the AOF before we dump the DB to disk
(out Aof (rewind)) # wipe the contents of the AOF
(unless (fork) (kv-save-db T) (bye))
(out *KV_aof (rewind)) # wipe the contents of the AOF
(unless (fork) (kv-save-db) (bye)) )
(kv-stat "rdb_last_bgsave_status" "Background saving started") ]
# Restore the in-memory database from entries stored in the DB file
@ -491,6 +469,7 @@
(kv-stat "aof_base_size" (car (info Filename)))
(kv-stat "loading_aof" 1)
(out Filename) # ensure the AOF exists
(in Filename
(while (read)
(inc '*ERROR_LINE)
@ -513,7 +492,7 @@
(use *ERROR_LINE
(zero *ERROR_LINE)
(when (and (info Filename) (gt0 (car @)))
(catch '("EOF Overrun" "Mismatched" "List expected" "Bad input")
(catch '(NIL)
(finally
(kv-read-error Type Filename)
(if (= "AOF" Type)
@ -521,10 +500,10 @@
(kv-restore-db Filename) ]
# Save a write command to the append-only log file with a timestamp and hash of the data
[de kv-save-aof (Request Aof) # Aof is a file descriptor
[de kv-save-aof (Request Aof) # Aof is a file
(when (member (car Request) *KV_write_commands)
(ctl *KV_aof_lock # try to obtain an exclusive lock
(out Aof (println (list (kv-timestamp T) (kv-hash Request) Request)))
(out (pack "+" Aof) (println (list (kv-timestamp T) (kv-hash Request) Request)))
(kv-stat "aof_last_write_status" (if @ "OK" "FAILED") ]
###
@ -592,7 +571,7 @@
# Process the message and send the result to the child over the named pipe
[de kv-sibling-job (Pid Msg)
(when *KV_persist (kv-save-aof Msg *Aof_desc)) # save the request to a log file first
(when *KV_persist (kv-save-aof Msg *KV_aof)) # save the request to a log file first
(let (Result (kv-process Pid Msg)
Pipe_child (pil "tmp/" *PPid "/pipe_child_" Pid) )
@ -651,24 +630,22 @@
(ifn (>= *Elapsed *KV_persist)
(abort (- *KV_persist *Elapsed) (kv-listen-sibling))
(setq *Start (time)) # restart the timer because it expired
(kv-bgsave-db *Aof_desc) ]
(kv-bgsave-db) ]
# Start the loop which listens for new messages
[de kv-sibling-loop ()
(use (*Aof_desc *Start *Elapsed)
(setq *Aof_desc (open *KV_aof)) # obtain a file descriptor for the AOF
(setq *Start (time)) # start the clock for the bgsave timer
(loop
(if *KV_persist
(kv-bgsave-timer)
(kv-listen-sibling) ]
# Restore the DB and AOF, then save it in the foreground (blocking)
(if *KV_persist
(use (*Start *Elapsed)
(setq *Start (time)) # start the clock for the bgsave timer
(loop (kv-bgsave-timer)) )
(loop (kv-listen-sibling) ]
# Restore the DB and AOF
[de kv-sibling-restore ()
(when *KV_persist
(kv-restore "DB" *KV_db)
(kv-restore "AOF" *KV_aof)
(kv-save-db) ]
(kv-bgsave-db) ] # perform an initial DB save on start
# Remove a locked process with SIGKILL
[de kv-remove-locked ()

2
libkvclient.l

@ -15,6 +15,8 @@
*KV_pass NIL
*KV_abort 60 # max time (in seconds) to wait for a message
*KV_uuid "7672FDB2-4D29-4F10-BA7C-8EAD0E29626E" # for server handshake, do not change
*KV_decrypt NIL
*KV_encrypt NIL
*KV_gpgid "secrets/gpg-id" ) # key name containing the GPG ID used to encrypt data
(off *KV_poll)

2
module.l

@ -1,6 +1,6 @@
[de APP_INFO
("name" "picolisp-kv")
("version" "0.16.0")
("version" "0.17.0")
("summary" "Redis-inspired in-memory key/value store written in PicoLisp")
("source" "https://github.com/aw/picolisp-kv")
("author" "Alexander Williams")

9
test.l

@ -13,18 +13,17 @@
# run key/value store tests with persistence
(chdir (pack (car (file)) "test/")
(call 'rm "-f" *KV_db (pack *KV_db ".old") *KV_aof *KV_aof_lock) # cleanup first
(unless (fork)
(exec '../server.l "--pass" *KV_pass "--port" *KV_port "--persist" *KV_persist) )
(kv-listen) )
(mapcar load (filter '((N) (sub? "test_kv" N)) (dir "."))) )
(kill (car (kids))) # ensure the server is stopped
(mapcar kill (kids))
(wait 1000)
(setq *KV_port (rand 40001 49999))
# run the client/server tests without persistence
#{ # NOTE: disabled integration tests
(chdir (pack (car (file)) "test/")
(call 'rm "-f" *KV_db (pack *KV_db ".old") *KV_aof *KV_aof_lock) # cleanup first
(unless (fork)
@ -32,6 +31,6 @@
(mapcar load (filter '((N) (sub? "test_cs" N)) (dir "."))) )
(kill (car (kids))) # ensure the server is stopped
}#
(report)

14
test/test_kv.l

@ -80,13 +80,11 @@
[de test-commands-bgsave ()
(kv-cmd-rpush "mylist" '("1" "2" "3" "4" "5" "6" "7" "8" "9" "10"))
(let KV_desc (open *KV_aof)
(kv-save-aof '("RPUSH" '("mylist" "1" "2" "3" "4" "5" "6" "7" "8" "9" "10")) KV_desc)
(close KV_desc) )
(kv-save-aof '("RPUSH" '("mylist" "1" "2" "3" "4" "5" "6" "7" "8" "9" "10")) *KV_aof)
(assert-equal "OK" (kv-cmd-get "%stats%/aof_last_write_status") "[BGSAVE] Should be OK for saving a valid AOF entry")
(kv-save-db-keys)
(kv-write-db)
(assert-equal 0 (kv-restore "DB" *KV_db) "[BGSAVE] Should return 0 when restoring the DB")
(kv-bgsave-db)
(call 'touch *KV_db)
(assert-equal 0 (kv-restore-db *KV_db) "[BGSAVE] Should return 0 when restoring the DB")
(out *KV_db_lock (prinl "12345"))
(assert-equal "Error: DB is locked for writing" (kv-locked?) "[BGSAVE] Should return an error if the DB is locked")
(call 'rm "-f" *KV_db_lock)
@ -94,8 +92,8 @@
(assert-kind-of 'String (kv-timestamp) "[BGSAVE] Should return a UNIX timestamp")
(kv-save-db-keys)
(assert-kind-of 'List (info *KV_db_tmp) "[BGSAVE] Should return a list if a temp DB exists after saving the keys")
(assert-t (kv-write-db) "[BGSAVE] Should write the DB to disk")
(assert-equal 0 (kv-rewrite-aof) "[BGSAVE] Should return 0 when writing the AOF to disk")
(assert-equal "Background saving started" (kv-bgsave-db) "[BGSAVE] Should write the DB to disk")
(assert-t (kv-rewrite-aof) "[BGSAVE] Should return 0 when writing the AOF to disk")
]
[de test-commands-exists ()

Loading…
Cancel
Save