# #+TITLE: Autodidactism at 700nm
# #+TITLE: Invisible Light through a Scanner Darkly
#+TITLE: Applied Apophenia
#+AUTHOR: Kevin Downey
#+AUTHOR: @hiredman@downey.family
#+AUTHOR: https://github.com/hiredman
#+PROPERTY: header-args :eval never-export
#+BEGIN_EXPORT html
#+END_EXPORT
#+NAME: foo
#+BEGIN_SRC javascript :exports none
function table_toggle_gadget(table) {
var rows = table.getElementsByTagName("tr");
var columnc = table.rows[0].cells.length;
var r = table.tBodies[0].insertRow();
var c = r.insertCell();
var fold = document.createTextNode("fold");
var unfold = document.createTextNode("unfold");
var button = document.createElement("button");
var state = true;
r.my_toggle_row = true;
button.onclick = function () {
var rows = table.getElementsByTagName("tr");
if(state) {
for(var ii = 0; ii < rows.length; ii++){
if(ii > 11){
rows[ii].style.display = "";
}
}
button.replaceChild(fold,unfold);
} else {
for(var ii = 0; ii < rows.length; ii++){
if(ii > 11 && !rows[ii].my_toggle_row){
console.log(rows[ii].style.display);
rows[ii].style.display = "none";
}
}
button.replaceChild(unfold,fold);
}
state = !state;
};
button.appendChild(unfold);
c.appendChild(button);
c.colSpan = columnc;
}
window.onload = function () {
var tables = document.getElementsByTagName("table");
for(var i = 0; i < tables.length; i++){
var rows = tables[i].getElementsByTagName("tr");
if(rows.length > 10) {
console.log(rows);
for(var ii = 0; ii < rows.length; ii++){
if(ii > 11){
console.log(rows[ii].style.display);
rows[ii].style.display = "none";
}
}
table_toggle_gadget(tables[i]);
}
}
};
#+END_SRC
#+name: inline-js
#+begin_src elisp :exports none :results html :var blk=""
(concat
"")
#+end_src
#+RESULTS: inline-js
#+begin_export html
#+end_export
#+call: inline-js("foo")
#+RESULTS:
#+begin_export html
#+end_export
# #+BEGIN_SRC elisp :exports results :results value html
# " Generated HTML code. "
# #+END_SRC
# #+RESULTS:
# #+begin_export html
# Generated HTML code.
# #+end_export
# bootstrapping org babel with clojure
#+NAME: boot-clojure
#+BEGIN_SRC emacs-lisp :exports none
(setq org-html-html5-fancy t
org-html-doctype "html5")
;; Run a clojure process listening on a socket
(let* ((clojure-program "clojure")
(buffer (get-buffer-create "*clojure*"))
(proc-alive (comint-check-proc buffer))
(process (get-buffer-process buffer)))
(unless proc-alive
(with-current-buffer buffer
(make-comint-in-buffer "Clojure" buffer clojure-program
nil
"-J-Dclojure.server.repl={:port,9457,:accept,clojure.core.server/repl}")))
;; (when buffer
;; (pop-to-buffer buffer))
)
#+END_SRC
#+RESULTS: boot-clojure
#+NAME: ob-clojure
#+BEGIN_SRC elisp :exports none
;; tell org babel to evaluate clojure code by sending it to that socket
(defun org-babel-execute:clojure (body params)
"Execute a block of clojure code with org-babel."
(org-babel-eval "nc localhost 9457" body))
#+END_SRC
#+RESULTS: ob-clojure
: org-babel-execute:clojure
#+NAME: boot-clojure2
#+BEGIN_SRC clojure :exports none
;; use org-babel to run some clojure code to start a slightly
;; different repl on a different port because I don't like how the
;; prompt is printed in the default
(future
(with-open [sock (java.net.ServerSocket. 32466)]
(loop []
(try
(with-open [s (.accept sock)
i (clojure.lang.LineNumberingPushbackReader.
((requiring-resolve 'clojure.java.io/reader) (.getInputStream s)))
o ((requiring-resolve 'clojure.java.io/writer) (.getOutputStream s))]
(binding [*in* i
,*out* o
,*err* o]
((requiring-resolve 'clojure.main/repl)
:prompt (constantly nil))))
(catch Throwable t
(prn t)))
(recur))))
#+END_SRC
#+RESULTS: boot-clojure2
#+begin_example
user=> user=> user=> user=> #object[clojure.core$future_call$reify__8544 0x428eaf3c {:status :failed, :val #error {
:cause "Address already in use"
:via
[{:type java.util.concurrent.ExecutionException
:message "java.net.BindException: Address already in use"
:at [java.util.concurrent.FutureTask report "FutureTask.java" 122]}
{:type java.net.BindException
:message "Address already in use"
:at [sun.nio.ch.Net bind0 "Net.java" -2]}]
:trace
[[sun.nio.ch.Net bind0 "Net.java" -2]
[sun.nio.ch.Net bind "Net.java" 556]
[sun.nio.ch.Net bind "Net.java" 545]
[sun.nio.ch.NioSocketImpl bind "NioSocketImpl.java" 629]
[java.net.ServerSocket bind "ServerSocket.java" 393]
[java.net.ServerSocket "ServerSocket.java" 275]
[java.net.ServerSocket "ServerSocket.java" 168]
[user$eval6343$fn__6344 invoke "NO_SOURCE_FILE" 4]
[clojure.core$binding_conveyor_fn$fn__5823 invoke "core.clj" 2047]
[clojure.lang.AFn call "AFn.java" 18]
[java.util.concurrent.FutureTask run "FutureTask.java" 317]
[java.util.concurrent.ThreadPoolExecutor runWorker "ThreadPoolExecutor.java" 1144]
[java.util.concurrent.ThreadPoolExecutor$Worker run "ThreadPoolExecutor.java" 642]
[java.lang.Thread run "Thread.java" 1623]]}}]
user=>
#+end_example
#+NAME: ob-clojure2
#+BEGIN_SRC elisp :exports none
;; redefine this to use the new port
(defun org-babel-execute:clojure (body params)
"Execute a block of clojure code with org-babel."
(message "params %s" params)
(let ((result (org-babel-eval "nc localhost 32466"
(if (string-match-p
(regexp-quote "table")
(cdr (assoc :results params)))
(if (string-match-p
;; TODO use :colnames yes
(regexp-quote "header")
(cdr (assoc :results params)))
(concat "(apply org-table ((juxt first rest) (do " body ")))")
(concat "(org-table (do " body "))"))
body))))
(org-babel-reassemble-table
result
(org-babel-pick-name (cdr (assq :colname-names params))
(cdr (assq :colnames params)))
(org-babel-pick-name (cdr (assq :rowname-names params))
(cdr (assq :rownames params))))))
#+END_SRC
#+RESULTS: ob-clojure2
: org-babel-execute:clojure
#+NAME: pretty-html
#+BEGIN_SRC clojure :exports none
;; will come in handy for histograms
(deftype UnquotedString [s]
Object
(toString [_]
s))
(defmethod print-method UnquotedString [s w]
(.write w (.-s s)))
(defn svg [thing]
(->UnquotedString
(cond
(string? thing) thing
(seq? thing) (apply str (map svg thing))
:else (let [[tag atrs? & content] thing
[attrs content] (if (map? atrs?)
[atrs? content]
[{} (cons atrs? content)])]
(str
#_(when (= tag :svg)
"")
"<" (name tag) " "
(apply str
(interpose \space
(for [[n v] attrs]
(str (name n) "="
(pr-str
(if (number? v)
(str (long v))
v))))))
">"
(apply str (map svg content))
"")))))
(defn org-table
([stuff]
(letfn [(column [row-id column-id data]
(let [s (str data)]
{:width (count s)
:text s
:row row-id
:col column-id}))
(row [row-id datums]
(map-indexed (partial column row-id) datums))
(table [datums]
(apply concat (map-indexed row datums)))]
(let [cells (table (seq stuff))
rows (group-by :row cells)
colums (group-by :col cells)
colum-widths
(into {}
(for [[col cells] colums]
[col (apply max (map :width cells))]))]
(->UnquotedString
(with-out-str
(println)
(dotimes [i (count rows)]
(let [row (get rows i)]
(print "|")
(doseq [cell row]
(print (:text cell))
(print "|"))
(when (> (count rows) (inc i))
(print "\n")))))))))
([headings stuff]
(->UnquotedString
(with-out-str
(println)
(print "|")
(doseq [i headings]
(print i)
(print "|"))
(println)
(print "|-")
(print (org-table stuff))))))
(def figure-counter (atom 0))
(defn figure [description fig]
(->UnquotedString
(with-out-str
(println "")
(println fig)
(println "")
(print "Fig." (swap! figure-counter inc) description)
(println "")
(println ""))))
#+END_SRC
#+RESULTS: pretty-html
: user.UnquotedString
: #object[clojure.lang.MultiFn 0x4e0891eb "clojure.lang.MultiFn@4e0891eb"]
: #'user/svg
: #'user/org-table
: #'user/figure-counter
: #'user/figure
# end of bootstrapping
* Motivation
#+BEGIN_QUOTE
The creature has a purpose, and his eyes are bright with it. [fn:keats]
#+END_QUOTE
Every home automation project eventually needs to control devices
that use an infrared remote. I have arrived that place.
For some devices there are lists of IR codes, numeric values and
combined with some timing information describe how to pulse an IR
led on and off to send commands to a device, available online. But
luckily for me I am having trouble finding IR codes for some of my
devices so I get to do this from scratch.
This involves setting up an IR receiver, and recording what it
receives for different button presses on remotes. In my case what I
get back is something like:
#+BEGIN_SRC clojure :exports results
[9136 4429 599 479 595 492 613 503 616 1581 588 534 597 493 598 1594 587 503 616 495 595 497 576 539 595 1584 579 1613 615 506 580 502 588 525 600 1581 580 518 609 507 596 1580 593 558 569 471 619 495 576 1623 605 521 570 494 594 527 596 494 596 493 578 517 611 503 596 498 595]
#+END_SRC
#+RESULTS:
: [9136 4429 599 479 595 492 613 503 616 1581 588 534 597 493 598 1594 587 503 616 495 595 497 576 539 595 1584 579 1613 615 506 580 502 588 525 600 1581 580 518 609 507 596 1580 593 558 569 471 619 495 576 1623 605 521 570 494 594 527 596 494 596 493 578 517 611 503 596 498 595]
This list of numbers that a receiver reports for a single button
press is a _message_. Buttons on a remote will (usually) result in
different messages. A message from the remote to the device tells
the device to do something. Different messages usually tell it to
do different things.
I can take this list of numbers and feed it backout through an IR
emitter to send the message to a device. Remote cloned, job done.
But this long list of numbers is annoying to work with, and
depending on the remote you can easily end up with 50+ of these
lists per remote. Each number in theory has the range of unsigned
16 bit number, so two bytes per for storage. And I want to to
reprogram my IR emitter on the fly. That is a lot of data to be
pushing into a poor little microntroller over a bluetooth radio.
Ideally I should be able to factor out commonality in these
messages to reduce their size, doing some kind of domain specific
compression.
There is a lot of noise is these numbers. So before attempting any
kind of more succint encoding of the information, I need to remove
the noise so the messages will _compress_ better. Removing noise
will make redundancy more evident.
I can use some structural knowledge of how these IR messages tend
to work to make a best guess at the amount of noise. After removing
the noise I can derive the _alphabet_ of _symbols_ that each remote
uses to write its messages, and generate an efficient binary
encoding for that alphabet.
To do this I'll need a nice representive sample of messages from
the target remote:
#+BEGIN_SRC clojure :exports both
(require '[clojure.java.io :as io])
(def pulse-file "/home/kevin/Downloads/whynter_ir_pulses.txt")
(defn load-messages [a-readable]
(with-open [i (io/reader a-readable)
pbr (clojure.lang.LineNumberingPushbackReader. i)]
(loop [pulses []]
;; the lists of pulses are actually written out by some
;; circuitpython code, but the clojure reader reads them just
;; fine.
(let [x (read pbr false :eof)]
(if (= x :eof)
pulses
(recur (conj pulses x)))))))
(def messages (load-messages (io/file pulse-file)))
(count messages)
#+END_SRC
#+RESULTS:
: nil
: #'user/pulse-file
: #'user/load-messages
: #'user/messages
: 347
* Some IR encoding basics
#+BEGIN_QUOTE
The visible world is no longer a reality and the unseen world no longer a dream. [fn:yeats]
#+END_QUOTE
IR messages tend to be of a fixed width, but my sample has a wide
variety of widths:
#+BEGIN_SRC clojure :exports both :results value table
(sort-by second (frequencies (map count messages)))
#+END_SRC
#+RESULTS:
| 7 | 1 |
| 55 | 1 |
| 15 | 1 |
| 99 | 1 |
| 41 | 1 |
| 195 | 1 |
| 61 | 1 |
| 6 | 1 |
| 64 | 1 |
| 200 | 1 |
| 5 | 1 |
| 53 | 1 |
| 30 | 1 |
| 71 | 1 |
| 8 | 1 |
| 49 | 1 |
| 65 | 2 |
| 21 | 2 |
| 3 | 2 |
| 57 | 3 |
| 9 | 3 |
| 69 | 4 |
| 19 | 6 |
| 135 | 12 |
| 1 | 29 |
| 67 | 268 |
These different messages widths might be the result of interference
and noise in my IR receiver setup (components on a breadboard) or
inattention on my part collecting the sample (pressing buttons too
quickly in succession, etc).
For my first step in removing noise, I will only consider
messages where the width matches the mode of message widths:
#+BEGIN_SRC clojure :exports both
(defn mode [xs]
(first (last (sort-by second (frequencies xs)))))
(def mode-width (mode (map count messages)))
(def messages-of-mode-width
(for [message messages
:when (= (count message) mode-width)]
message))
(count messages-of-mode-width)
#+END_SRC
#+RESULTS:
: #'user/mode
: #'user/mode-width
: #'user/messages-of-mode-width
: 268
** TODO Entropy
I can check my progress removing noise by measuring the [[https://en.wikipedia.org/wiki/Entropy_(information_theory)][entropy]] of the
values that make up my message:
#+BEGIN_SRC clojure :exports code
(defn log [a b]
(/ (Math/log a) (Math/log b)))
(defn entropy [xs]
(let [X (distinct xs)
c (count xs)
p (into {} (map (fn [[k v]] [k (/ v c)])) (frequencies xs))]
(- (apply + 0 (for [x X] (* (p x) (log (p x) 2)))))))
(defn parts-of [messages]
(apply concat messages))
#+END_SRC
#+RESULTS:
: #'user/log
: #'user/entropy
: #'user/parts-of
#+BEGIN_SRC clojure :exports both
(entropy (repeat 100 0))
(entropy (repeat 100 1))
(entropy (concat (repeat 50 0) (repeat 50 1)))
#+END_SRC
#+RESULTS:
: -0.0
: -0.0
: 1.0
The entropy of a binary signal (only made of two components) will
always be between 0 and 1. My signal is decidedly analog at this
point, and noisy.
#+BEGIN_SRC clojure :exports both
(entropy (parts-of messages))
(entropy (parts-of messages-of-mode-width))
(- (entropy (parts-of messages)) (entropy (parts-of messages-of-mode-width)))
#+END_SRC
#+RESULTS:
: 7.612712682910637
: 7.561958310958734
: 0.05075437195190258
A slight improvement. Perfect.
- clarify entropy of what
- entropy of the parts of messages
** Ook-Ook
Each message is recorded as a list of durations in
microseconds. This format is describes a message as [[https://en.wikipedia.org/wiki/On%E2%80%93off_keying][On-off Keying]]. The
first duration in a message is how long to turn your transmitter
on. The second duration is how long to turn the transmitter
off. Turning the transmitter on is called a _mark_, turning it off is
called a _space_. The message is composed of these pairs of marks and
spaces of varying durations up until the last duration in a message,
which has to be a mark, because the final space is of indefinite
length and not included in the list.
For many IR message formats the first mark and space are special,
they are what tell an IR receiver to wake up and listen, are a
constant value, and don't convey any data.
#+BEGIN_SRC clojure :exports code
(def head-mark-index 0)
(def head-space-index 1)
#+END_SRC
#+RESULTS:
: #'user/head-mark-index
: #'user/head-space-index
Because the final mark in a message doesn't get an acompanying
space, it is often treated as special end of message mark.
#+BEGIN_SRC clojure :exports code
(def tail-mark-index (dec mode-width))
#+END_SRC
#+RESULTS:
: #'user/tail-mark-index
The rest of the message between the head space and tail mark are
pairs of data marks and spaces:
#+BEGIN_SRC clojure :exports code
(def data-mark-indices (range (inc head-space-index) tail-mark-index 2))
(def data-space-indices (range (+ 2 head-space-index) tail-mark-index 2))
(def data-indices (sort (concat data-mark-indices data-space-indices)))
#+END_SRC
#+RESULTS:
: #'user/data-mark-indices
: #'user/data-space-indices
: #'user/data-indices
Next I can verify my assumptions about the head, tail, and data
parts of the message by looking at the variance of the durations in
those parts of the messages.
#+BEGIN_SRC clojure :exports both :results value table
(defn mean [x]
(/ (apply + 0 x) (double (count x))))
(defn variance [x]
(let [m (mean x)]
(/ (apply + 0 (for [i x] (Math/pow (- i m) 2)))
(count x))))
(for [[name durs] [[:head-marks (for [message messages-of-mode-width]
(nth message head-mark-index))]
[:head-spaces (for [message messages-of-mode-width]
(nth message head-space-index))]
[:tail-marks (for [message messages-of-mode-width]
(nth message tail-mark-index))]
[:data-marks (for [message messages-of-mode-width
i data-mark-indices]
(nth message i))]
[:data-spaces (for [message messages-of-mode-width
i data-space-indices]
(nth message i))]]]
[name (variance durs)])
#+END_SRC
#+RESULTS:
| :head-marks | 1353.8603252394746 |
| :head-spaces | 1029.6314741590552 |
| :tail-marks | 527.0302962797954 |
| :data-marks | 355.5791307136194 |
| :data-spaces | 275725.7342396329 |
Uh oh, the data-marks variance is lower the variance of anything
else, which doesn't seem like it would be the case if the head and
tail are constants. However the data marks and spaces are handled
as pairs, and the variance of data-spaces dwarfs everything else. I
think this means my head and tail assumption is correct, and shows
that most of the signal/noise in these messages exists in the
spaces.
So each message is something like:
#+BEGIN_SRC clojure :exports code
'[head-mark head-space pairs-of-data-marks-and-spaces... tail-mark]
#+END_SRC
#+RESULTS:
: [head-mark head-space pairs-of-data-marks-and-spaces... tail-mark]
** TODO NEC codes
- most popular ir protocol
- format
- 9 ms mark
- 4.5 ms space
- 8 bit address
- 8 bit inverse address
- 8 bit command
- 8 bit inverse of command
- 562.5 microsecond end of message
#+BEGIN_SRC clojure :exports both
median-head-mark
median-head-space
median-tail-mark
#+END_SRC
#+RESULTS:
: 9207
: 4433
: 682
The median values don't exactly match the NEC code spec, but are
close. Once we get the signal to something like bits we can check for
things like the address being followed by the inverse of itself.
As a coda to IR basics, it should be mentioned that IR signals are
modulated, typically at 38khz. This means when you send a
mark, the IR transmitter isn't just on, but turning on and off at
38khz. This helps avoid interference from other sources of IR in
the environment, but doesn't really effect this analysis.
* TODO Analysis
** Denoising
Since I know that the head mark is a constant value, I can use the
head mark to estimate the amount of noise in the durations. I can
do that by taking half the range of the durations of head marks.
#+BEGIN_SRC clojure :exports code
(defn range-stat [xs]
(apply - ((juxt last first) (sort xs))))
(defn estimate-noise [messages column]
(range-stat
(for [message messages]
(nth message column))))
(def noise-est
(estimate-noise messages-of-mode-width head-mark-index))
#+END_SRC
#+RESULTS:
: #'user/range-stat
: #'user/estimate-noise
: #'user/noise-est
I can clean up the messages by factoring the estimated noise out of
the durations:
#+BEGIN_SRC clojure :exports code
(defn clean-duration [dur]
(* noise-est (Math/round (/ dur (double noise-est)))))
(def cleaned-messages (for [message messages-of-mode-width] (mapv clean-duration message)))
#+END_SRC
#+RESULTS:
: #'user/clean-duration
: #'user/cleaned-messages
Then some quick progress checks:
#+BEGIN_SRC clojure :exports both
(- (entropy (parts-of messages-of-mode-width))
(entropy (parts-of cleaned-messages)))
#+END_SRC
#+RESULTS:
: 5.981984617731911
Nice.
Since the head and tail are constant, I can replace those with
median values for a quick win:
#+BEGIN_SRC clojure :exports both
(defn median [xs]
(nth (sort xs) (quot (count xs) 2)))
(def median-head-mark
(median
(for [message cleaned-messages]
(nth message head-mark-index))))
(def median-head-space
(median
(for [message cleaned-messages]
(nth message head-space-index))))
(def median-tail-mark
(median
(for [message cleaned-messages]
(nth message tail-mark-index))))
(def constant-head-and-tail-messages
(for [message cleaned-messages]
(assoc message
head-mark-index median-head-mark
head-space-index median-head-space
tail-mark-index median-tail-mark)))
(- (entropy (parts-of cleaned-messages))
(entropy (parts-of constant-head-and-tail-messages)))
#+END_SRC
#+RESULTS:
: #'user/median
: #'user/median-head-mark
: #'user/median-head-space
: #'user/median-tail-mark
: #'user/constant-head-and-tail-messages
: 0.0016598727160686
A slight improvement
** TODO Binning
I'll assume that the pairs of data marks and spaces are used to
represent binary digits. That means there should be at maximum two
different data mark values, and two different data space values. At
a minimum one of the two needs two values. I'll look at the
distrbution of values for data marks and spaces to see if the way
the data looks confirms that:
#+BEGIN_SRC clojure :exports both
(def data-marks
(for [message constant-head-and-tail-messages
i data-mark-indices]
(nth message i)))
(def data-spaces
(for [message constant-head-and-tail-messages
i data-space-indices]
(nth message i)))
;; background color TODO
(defn histogram3 [xs]
(let [f (frequencies xs)
bins (mapv first (sort-by first f))
width 600
height 400
bin-width (/ width (count bins))]
(svg
[:svg {:width width :height height}
[:rect
{:x 0
:y 0
:width width
:height height
:fill "white"}]
(doall
(for [[i b] (map-indexed vector bins)
:let [h (* (f b) (/ (* 0.95 height) (apply max xs)))]]
[:rect
{:x (* i bin-width)
:y (- height h)
:width (* 0.8 bin-width)
:height h
:fill "rgb(59 130 246 / 0.5)"}]))])))
#+END_SRC
#+RESULTS:
: #'user/data-marks
: #'user/data-spaces
: #'user/histogram3
#+BEGIN_SRC clojure :exports both :results value html
(figure "Histogram of Data Marks" (histogram3 data-marks))
#+END_SRC
#+RESULTS:
#+begin_export html
Fig. 1 Histogram of Data Marks
#+end_export
Looks modal.
#+BEGIN_SRC clojure :exports both
(def data-mark-modes [(mode data-marks)])
#+END_SRC
#+RESULTS:
: #'user/data-mark-modes
#+name: data-spaces-histogram
#+BEGIN_SRC clojure :exports both :results value html
(figure "Histogram of Data Spaces" (histogram3 data-spaces))
#+END_SRC
#+RESULTS: data-spaces-histogram
#+begin_export html
Fig. 2 Histogram of Data Spaces
#+end_export
Looks bimodal.
#+BEGIN_SRC clojure :exports both
(def data-space-modes (map first (take 2 (reverse (sort-by second (frequencies data-spaces))))))
#+END_SRC
#+RESULTS:
: #'user/data-space-modes
Now that I have modal values for data marks and spaces, I can
replace data marks and spaces with their nearest modal value.
#+BEGIN_SRC clojure :exports both
(defn replace-with-closest [messages replacements columns]
(for [message messages]
(reduce
(fn [m i]
(assoc m i (first (first (sort-by second (for [ii replacements]
[ii (Math/abs (- (nth m i) ii))]))))))
(vec message)
columns)))
(def modal-messages
(as-> constant-head-and-tail-messages %
(replace-with-closest % data-mark-modes data-mark-indices)
(replace-with-closest % data-space-modes data-space-indices)))
#+END_SRC
#+RESULTS:
: #'user/replace-with-closest
: #'user/modal-messages
*** TODO WTF is up with this progress check?
Progress check:
#+BEGIN_SRC clojure :exports both
(- (entropy (parts-of constant-head-and-tail-messages)) (entropy (parts-of modal-messages)))
#+END_SRC
#+RESULTS:
: -0.06359679902433468
#+BEGIN_SRC clojure :exports both :results value table
(frequencies (parts-of constant-head-and-tail-messages))
#+END_SRC
#+RESULTS:
| 9207 | 268 |
| 4433 | 268 |
| 682 | 10204 |
| 341 | 4167 |
| 1705 | 3027 |
| 1364 | 22 |
#+BEGIN_SRC clojure :exports both :results value table
(frequencies (parts-of modal-messages))
#+END_SRC
#+RESULTS:
| 9207 | 268 |
| 4433 | 268 |
| 682 | 8844 |
| 341 | 5527 |
| 1705 | 3049 |
#+BEGIN_SRC clojure :exports both :results value table
(frequencies
(for [message constant-head-and-tail-messages
i (map message data-mark-indices)]
i))
#+END_SRC
#+RESULTS:
| 682 | 8553 |
| 341 | 23 |
#+BEGIN_SRC clojure :exports both :results value table
(frequencies
(for [message modal-messages
i (map message data-mark-indices)]
i))
#+END_SRC
#+RESULTS:
| 682 | 8576 |
#+BEGIN_SRC clojure :exports both :results value table
(frequencies
(for [message constant-head-and-tail-messages
i (map message data-space-indices)]
i))
#+END_SRC
#+RESULTS:
| 341 | 4144 |
| 1705 | 3027 |
| 682 | 1383 |
| 1364 | 22 |
#+BEGIN_SRC clojure :exports both :results value table
(frequencies
(for [message modal-messages
i (map message data-space-indices)]
i))
#+END_SRC
#+RESULTS:
| 341 | 5527 |
| 1705 | 3049 |
#+BEGIN_SRC clojure :exports both
(- (entropy (remove #{682} (parts-of constant-head-and-tail-messages))) (entropy (remove #{682} (parts-of modal-messages))))
#+END_SRC
#+RESULTS:
: 0.1055483101158654
** TODO Occult Sigils
#+BEGIN_QUOTE
The world is full of magic things, patiently waiting for our senses to grow sharper. [fn:yeats]
#+END_QUOTE
Each message is composed of pairs of marks and spaces. I can pair
up the marks and spaces, and assign a letter to each pair to create
an alphabet for expressing messages.
#+BEGIN_SRC clojure :exports both
(def paired-messages
(for [message modal-messages]
(vec (partition-all 2 message))))
(- (entropy (parts-of modal-messages)) (entropy (parts-of paired-messages)))
#+END_SRC
#+RESULTS:
: #'user/paired-messages
: 0.37665243454918507
#+BEGIN_SRC clojure :exports both
(def pair-usages (frequencies (apply concat paired-messages)))
(def alphabet
(into {}
(map
(fn [[pair _] letter] [(vec pair) letter])
(sort-by (comp - second) pair-usages)
;; some utf16 stuff
(for [point (range 0x1f736 0x1f75d)]
(str
(doto (StringBuilder.)
(.appendCodePoint point)))))))
(require '[clojure.set :as set])
(def alphabet-inverse (set/map-invert alphabet))
#+END_SRC
#+RESULTS:
: #'user/pair-usages
: #'user/alphabet
: nil
: #'user/alphabet-inverse
#+BEGIN_SRC clojure :exports both :results value table
alphabet
#+END_SRC
#+RESULTS:
| [682 341] | πΆ [fn:stargate] |
| [682 1705] | π· |
| [9207 4433] | πΈ |
| [682] | πΉ |
Given the alphabet I can re-express messages in terms of the alphabet:
#+BEGIN_SRC clojure :exports both
(def alphabet-messages
(for [message paired-messages]
(mapv alphabet message)))
(def head-letters (set (map first alphabet-messages)))
(def tail-letters (set (map last alphabet-messages)))
(def data-letters
(-> (set (parts-of alphabet-messages))
(set/difference head-letters tail-letters)))
#+END_SRC
#+RESULTS:
: #'user/alphabet-messages
: #'user/head-letters
: #'user/tail-letters
: #'user/data-letters
Now that the messages are composed of short symbols, even more internal structure is readily apparent:
#+name: example_alphabet
#+BEGIN_SRC clojure :exports both :results value table
alphabet-messages
#+END_SRC
# #+BEGIN_EXPORT html
#
# #+END_EXPORT
I can look for constant columns in my sample of messages by taking the
entropy of each column.
#+BEGIN_SRC clojure :exports both :results value table header
(cons
(list*
:index
:entropy
(sort (vals alphabet)))
(for [i (range (count (first alphabet-messages)))
:let [letters (for [message alphabet-messages]
(message i))
f (frequencies letters)]]
(list*
i
(entropy letters)
(map #(f % 0) (sort (vals alphabet))))))
#+END_SRC
#+RESULTS:
| :index | :entropy | πΆ | π· | πΈ | πΉ |
|--------+---------------------+-----+-----+-----+-----|
| 0 | -0.0 | 0 | 0 | 268 | 0 |
| 1 | -0.0 | 268 | 0 | 0 | 0 |
| 2 | -0.0 | 268 | 0 | 0 | 0 |
| 3 | -0.0 | 268 | 0 | 0 | 0 |
| 4 | -0.0 | 0 | 268 | 0 | 0 |
| 5 | -0.0 | 268 | 0 | 0 | 0 |
| 6 | -0.0 | 268 | 0 | 0 | 0 |
| 7 | -0.0 | 0 | 268 | 0 | 0 |
| 8 | -0.0 | 268 | 0 | 0 | 0 |
| 9 | -0.0 | 268 | 0 | 0 | 0 |
| 10 | 0.19364280468389863 | 260 | 8 | 0 | 0 |
| 11 | 0.22983578430824564 | 258 | 10 | 0 | 0 |
| 12 | 0.35525067854925235 | 18 | 250 | 0 | 0 |
| 13 | 0.4349734800747425 | 24 | 244 | 0 | 0 |
| 14 | -0.0 | 268 | 0 | 0 | 0 |
| 15 | 0.31123570787647303 | 253 | 15 | 0 | 0 |
| 16 | 0.21204937133912227 | 259 | 9 | 0 | 0 |
| 17 | 0.3409238906708339 | 17 | 251 | 0 | 0 |
| 18 | 0.3409238906708339 | 251 | 17 | 0 | 0 |
| 19 | 0.9313738396976812 | 175 | 93 | 0 | 0 |
| 20 | 0.9682710238495325 | 106 | 162 | 0 | 0 |
| 21 | 0.9610417290755667 | 165 | 103 | 0 | 0 |
| 22 | 0.459461070711055 | 26 | 242 | 0 | 0 |
| 23 | 0.4349734800747425 | 244 | 24 | 0 | 0 |
| 24 | 0.2637774366941386 | 12 | 256 | 0 | 0 |
| 25 | 0.9985532860835362 | 128 | 140 | 0 | 0 |
| 26 | 0.9207633345559458 | 178 | 90 | 0 | 0 |
| 27 | 0.9967435309724345 | 125 | 143 | 0 | 0 |
| 28 | 0.9705123810496528 | 161 | 107 | 0 | 0 |
| 29 | 0.9705123810496528 | 161 | 107 | 0 | 0 |
| 30 | 0.702095496871958 | 217 | 51 | 0 | 0 |
| 31 | 0.8652254972177151 | 77 | 191 | 0 | 0 |
| 32 | -0.0 | 268 | 0 | 0 | 0 |
| 33 | -0.0 | 0 | 0 | 0 | 268 |
The common prefix of _πΈπΆπΆπΆπ·πΆπΆπ·πΆ_ starts with _πΈ_ which is the head
mark and space, and the 8 following symbols are likely some kind of
identifier, model or manufacturer.
*** TODO Are these NEC codes?
*** TODO Binary
And now a fork in the road. Two choices, a binary decision. Do I
keep the head and tail symbols in band, or do I only deal with the
data in our encoding and cover those values out of band. In either
case I'll need some kind of dictionary for mapping bits to
durations.
The information I need for each pulse is:
1. the 1 and 0s of the data bits
2. the duration of the mark and space for 1
3. the duration of the mark and space for 0
4. the duration of the head mark
5. the duration of the head space
6. the duration of the tail mark
#+begin_src clojure :exports both
(apply max
(for [message modal-messages]
(->> (map message data-indices)
(partition-all 2)
(count))))
#+end_src
#+RESULTS:
: 32
Looks like each message has 32 data bits. That combined with the
_uint16s_ for each duration needed ends up with something like 14
bytes per message. With some squeezing maybe I can get it down to 9
bytes, which is still too many. _uint64_ (8 bytes, 64 bits) is as
big as I would like to go. Looks like the duration information is
moving out of band.
#+begin_src clojure :exports both
(def headless-tailess-messages
(for [message alphabet-messages]
(vec (rest (butlast message)))))
(entropy (parts-of headless-tailess-messages))
(def pair-to-bits
(into {}
(map
(fn [letter digit]
[letter digit])
data-letters
(range))))
#+end_src
#+RESULTS:
: #'user/headless-tailess-messages
: 0.9389075527190375
: #'user/pair-to-bits
#+begin_src clojure :exports both :results value table
pair-to-bits
#+end_src
#+RESULTS:
| πΆ | 0 |
| π· | 1 |
** TODO Ints
** TODO look for codes elsewhere
* TODO Synthesis
** TODO Build a Model
*** TODO OOK->int
*** TODO int<-OOK
* TODO Evaluation
** TODO Does it work with the original remote?
** TODO Does it work with other remotes?
** TODO Generating new messages
* TODO Postscript
#+BEGIN_QUOTE
We have woven a web, you and I, attached to this world but a separate world of our own invention. [fn:keats]
#+END_QUOTE
** TODO credit quotes
** TODO Bootstrapping
*** TODO Org Source
* TODO Parked
#+BEGIN_SRC clojure :exports both :results value html
(svg
[:svg {:width 300 :height 200}
[:rect {:width "100%" :height "100%" :fill "green"}]])
#+END_SRC
#+RESULTS:
#+begin_export html
#+end_export
[fn:keats] John Keats
[fn:yeats] W. B. Yeats
[fn:stargate] Earth, the point of origin.