Linear Regression auto-mpg - Clojure / Incanter

(use 'incanter.core)
(import (java.io BufferedReader BufferedWriter InputStreamReader FileReader FileWriter PrintWriter))

(defn linear-regression [X t]
  (let [X_T (trans X)]
    (mmult (solve (mmult X_T X)) X_T t)))

(defn remove-col [coll-of-colls n]
  (map #(concat (take n %) (drop (inc n) %)) coll-of-colls))

(defn odd-items [coll] (map first (partition 1 2 coll)))
(defn even-items [coll] (odd-items (rest coll)))

(defn read-data [file-name feature-size]
  (with-open [rdr (BufferedReader. (FileReader. file-name))]
    (reduce
     (fn [acc line]
       (if (re-matches #".*\?.*" line)
         acc
         (if-let [data-seq (re-seq #"\S+" (re-find #"^[^\"]+" line))]
           (let [[features target] (split-at feature-size (map #(Double/parseDouble %) data-seq))]
             (assoc (assoc acc 0 (conj (get acc 0) (vec features))) 1 (conj (get acc 1) (vec (conj target -1)))))
           acc)))
     [[] []] (line-seq rdr))))
       
(def auto-data (read-data "../RL/Marsland Machine Learning/auto-mpg.data" 1))

(def inputs (second auto-data))
(def targets (first auto-data))

; SUM-OF-SQUARES ERROR
;(let [inputs (remove-col (remove-col inputs 1) 4) ; data cleanup
(let [inputs (remove-col inputs 1) ; data cleanup
      beta (trans (linear-regression (odd-items inputs) (odd-items targets)))]
  (reduce + (map #(Math/pow (- %1 (first %2)) 2) (map #(mmult beta %) (even-items inputs)) (even-items targets))))

Linear Regression Clojure / Incanter

(use 'incanter.core)

(def X [[-1 0 0] [-1 0 1] [-1 1 0] [-1 1 1]])
(def t [[0] [1] [1] [1]]) ; OR
(def t [[0] [0] [0] [1]]) ; AND
(def t [[0] [1] [1] [0]]) ; XOR
(def t [[1] [0] [0] [0]]) ; NOR
(def t [[1] [1] [1] [0]]) ; NAND

(defn linear-regression [X t]
  (let [X_T (trans X)]
    (mmult (solve (mmult X_T X)) X_T t)))

(doseq [m (map #(mmult (trans (linear-regression X t)) %) X)] (prn m))

Perceptron Common Lisp

The algorithm:

(defun g (sum)
  (if (> sum 0) 1 0))

(defun perceptron-predict (weights input g)
  (mapcar #'(lambda (weights-k) (funcall g (reduce #'+ (mapcar #'* weights-k input)))) weights))

(defun perceptron-training-step (weights input target eta g)
  (let ((ys (perceptron-predict weights input g)))
    (mapcar #'(lambda (w_i_k y_k t_k)
                (mapcar #'(lambda (w x) (+ w (* eta (- t_k y_k) x))) w_i_k input))
            weights ys target)))

(defun perceptron-train (weights inputs targets eta g max-iteration)
  (format t "~A~%" (mapcar #'(lambda (input) (perceptron-predict weights input g)) inputs))
  (let ((new-weights
         (reduce #'(lambda (w input-target)
                    (perceptron-training-step w (first input-target) (second input-target) eta g))
                 (mapcar #'list inputs targets) :initial-value weights)))
    (format t "ITERATION: ~A / (max iterations: ~A)~%" new-weights max-iteration)
    (if (or (< max-iteration 2) (equal weights new-weights))
        (progn
          (format t "~A~%" (mapcar #'(lambda (input) (perceptron-predict new-weights input g)) inputs))
          new-weights)
        (perceptron-train new-weights inputs targets eta g (- max-iteration 1)))))

Run on OR:

* (perceptron-train '((3 2 1)) '((-1 0 0) (-1 0 1) (-1 1 0) (-1 1 1)) '((0) (1) (1) (1)) 0.25 #'g 10)

((0) (0) (0) (0))
ITERATION: ((2.5 2.25 1.25)) / (max iterations: 10)
((0) (0) (0) (1))
ITERATION: ((2.0 2.5 1.5)) / (max iterations: 9)
((0) (0) (1) (1))
ITERATION: ((1.75 2.5 1.75)) / (max iterations: 8)
((0) (0) (1) (1))
ITERATION: ((1.5 2.5 2.0)) / (max iterations: 7)
((0) (1) (1) (1))
ITERATION: ((1.5 2.5 2.0)) / (max iterations: 6)
((0) (1) (1) (1))
((1.5 2.5 2.0))

Run on XOR:

* (perceptron-train '((3 2 1)) '((-1 0 0) (-1 0 1) (-1 1 0) (-1 1 1)) '((0) (1) (1) (0)) 0.25 #'g 10)

((0) (0) (0) (0))
ITERATION: ((2.75 2.0 1.0)) / (max iterations: 10)
((0) (0) (0) (1))
ITERATION: ((2.5 2.0 1.0)) / (max iterations: 9)
((0) (0) (0) (1))
ITERATION: ((2.25 2.0 1.0)) / (max iterations: 8)
((0) (0) (0) (1))
ITERATION: ((2.0 2.0 1.0)) / (max iterations: 7)
((0) (0) (0) (1))
ITERATION: ((2.0 1.75 1.0)) / (max iterations: 6)
((0) (0) (0) (1))
ITERATION: ((1.75 1.75 1.0)) / (max iterations: 5)
((0) (0) (0) (1))
ITERATION: ((1.75 1.5 1.0)) / (max iterations: 4)
((0) (0) (0) (1))
ITERATION: ((1.5 1.5 1.0)) / (max iterations: 3)
((0) (0) (0) (1))
ITERATION: ((1.5 1.25 1.0)) / (max iterations: 2)
((0) (0) (0) (1))
ITERATION: ((1.25 1.25 1.0)) / (max iterations: 1)
((0) (0) (0) (1))
((1.25 1.25 1.0))

Perceptron Clojure

The algorithm:

(defn g [sum]
  (if (> sum 0) 1 0))

(defn perceptron-predict
  [weights input g]
  (map #(g (reduce + 0 (map * % input))) weights))

(defn perceptron-training-step
  [weights input target eta g]
  (let [ys (perceptron-predict weights input g)]
    (map (fn [w_i_k y_k t_k]
           (map #(+ %1 (* eta (- t_k y_k) %2)) w_i_k input))
         weights ys target)))

(defn perceptron-train
  [weights inputs targets eta g max-iteration]
  (prn (map #(perceptron-predict weights % g) inputs))
  (let [new-weights
        (reduce (fn [w [input target]] (perceptron-training-step w input target eta g)) weights (map list inputs targets))]
    (println (str "ITERATION: " (vec new-weights) " / (max iterations: " max-iteration ")"))
    (if (or (< max-iteration 2) (= weights new-weights))
      (do
        (prn (map #(perceptron-predict new-weights % g) inputs))
        new-weights)
      (recur new-weights inputs targets eta g (dec max-iteration)))))

Let's see if it can approximate a simple OR function (the '-1' in the inputs is the bias):

user=> (perceptron-train [[3 2 1]] [[-1 0 0] [-1 0 1] [-1 1 0] [-1 1 1]] [[0] [1] [1] [1]] 0.25 g 10)
((0) (0) (0) (0))
ITERATION: [(2.5 2.25 1.25)] / (max iterations: 10)
((0) (0) (0) (1))
ITERATION: [(2.0 2.5 1.5)] / (max iterations: 9)
((0) (0) (1) (1))
ITERATION: [(1.75 2.5 1.75)] / (max iterations: 8)
((0) (0) (1) (1))
ITERATION: [(1.5 2.5 2.0)] / (max iterations: 7)
((0) (1) (1) (1))
ITERATION: [(1.5 2.5 2.0)] / (max iterations: 6)
((0) (1) (1) (1))
((1.5 2.5 2.0))

What about XOR? It's not linearly separable, so it should not converge.

user=> (perceptron-train [[3 2 1]] [[-1 0 0] [-1 0 1] [-1 1 0] [-1 1 1]] [[0] [1] [1] [0]] 0.25 g 10)
((0) (0) (0) (0))
ITERATION: [(2.75 2.0 1.0)] / (max iterations: 10)
((0) (0) (0) (1))
ITERATION: [(2.5 2.0 1.0)] / (max iterations: 9)
((0) (0) (0) (1))
ITERATION: [(2.25 2.0 1.0)] / (max iterations: 8)
((0) (0) (0) (1))
ITERATION: [(2.0 2.0 1.0)] / (max iterations: 7)
((0) (0) (0) (1))
ITERATION: [(2.0 1.75 1.0)] / (max iterations: 6)
((0) (0) (0) (1))
ITERATION: [(1.75 1.75 1.0)] / (max iterations: 5)
((0) (0) (0) (1))
ITERATION: [(1.75 1.5 1.0)] / (max iterations: 4)
((0) (0) (0) (1))
ITERATION: [(1.5 1.5 1.0)] / (max iterations: 3)
((0) (0) (0) (1))
ITERATION: [(1.5 1.25 1.0)] / (max iterations: 2)
((0) (0) (0) (1))
ITERATION: [(1.25 1.25 1.0)] / (max iterations: 1)
((0) (0) (0) (1))
((1.25 1.25 1.0))