"""Fully connected networks, explicit backpropagation and Adam. No pretrained data.""" import copy import math import numpy as np ACTIVATIONS = ("linear", "relu", "sigmoid", "tanh") DEFAULT = {"sizes": [2, 16, 16, 1], "activations": ["relu", "relu", "linear"], "seed": 43, "learning_rate": 0.003, "scale": 20.0} DEFAULT_TRAIN = [{"x": [1, 1], "y": [2]}, {"x": [1, 2], "y": [3]}] DEFAULT_TEST = DEFAULT_TRAIN + [{"x": [2, 1], "y": [3]}] def integer(value, name, lo, hi): if isinstance(value, bool) or not isinstance(value, (int, float)) or int(value) != value: raise ValueError(f"{name}: ganze Zahl erforderlich.") value = int(value) if not lo <= value <= hi: raise ValueError(f"{name}: zulässiger Bereich {lo} bis {hi}.") return value def finite(value, name, lo=None, hi=None): if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value): raise ValueError(f"{name}: endliche Zahl erforderlich.") value = float(value) if (lo is not None and value < lo) or (hi is not None and value > hi): raise ValueError(f"{name}: Wert außerhalb des zulässigen Bereichs.") return value def config_valid(config): if not isinstance(config, dict): raise ValueError("Modellkonfiguration fehlt.") sizes = config.get("sizes", []) if not isinstance(sizes, list) or not 2 <= len(sizes) <= 10: raise ValueError("Das Netz benötigt 2 bis 10 Schichten einschließlich Ein- und Ausgabe.") sizes = [integer(n, "Neuronenzahl", 1, 128) for n in sizes] if sizes[0] > 32 or sizes[-1] > 32: raise ValueError("Bis zu 32 Eingaben und 32 Ausgaben sind möglich.") acts = config.get("activations", []) if not isinstance(acts, list) or len(acts) != len(sizes) - 1 or any(a not in ACTIVATIONS for a in acts): raise ValueError("Für jede berechnende Schicht eine gültige Aktivierung wählen.") count = sum((a + 1) * b for a, b in zip(sizes, sizes[1:])) if count > 100000: raise ValueError("Dieses Lernlabor unterstützt bis zu 100.000 Parameter.") return dict(sizes=sizes, activations=acts[:], seed=integer(config.get("seed", 43), "Zufallsstart", 0, 2**32 - 1), learning_rate=finite(config.get("learning_rate", .003), "Lernrate", 1e-7, 1), scale=finite(config.get("scale", 20), "Skalierung", .001, 1e6)) def data_valid(data, config, allow_empty=False): if not isinstance(data, list) or len(data) > 100 or (not allow_empty and not data): raise ValueError("Eine Trainingsliste benötigt 1 bis 100 Aufgaben; die Testliste höchstens 100.") result = [] for case in data: if not isinstance(case, dict) or not isinstance(case.get("x"), list) or not isinstance(case.get("y"), list): raise ValueError("Jede Aufgabe benötigt Eingaben und Zielwerte.") if len(case["x"]) != config["sizes"][0] or len(case["y"]) != config["sizes"][-1]: raise ValueError("Die Aufgaben passen nicht zur Ein- oder Ausgabeschicht.") result.append(dict(x=[finite(x, "Eingabe", -1e6, 1e6) for x in case["x"]], y=[finite(y, "Zielwert", -1e6, 1e6) for y in case["y"]])) return result def criterion_valid(value): if not isinstance(value, dict): raise ValueError("Erfolgskriterium fehlt.") mode = value.get("mode", "rounded") if mode not in ("rounded", "tolerance"): raise ValueError("Unbekanntes Erfolgskriterium.") return dict(mode=mode, tolerance=finite(value.get("tolerance", .01), "Toleranz", 1e-12, 1e6)) def activate(z, kind): if kind == "linear": return z if kind == "relu": return np.maximum(z, 0) if kind == "tanh": return np.tanh(z) # Stable sigmoid, including large negative inputs. out = np.empty_like(z) positive = z >= 0 out[positive] = 1 / (1 + np.exp(-z[positive])) expz = np.exp(z[~positive]) out[~positive] = expz / (1 + expz) return out def derivative(z, a, kind): if kind == "linear": return np.ones_like(z) if kind == "relu": return (z > 0).astype(float) if kind == "tanh": return 1 - a*a return a * (1-a) class Network: def __init__(self, config): self.config = config_valid(config) rng = np.random.default_rng(self.config["seed"]) self.p = [] for a, b in zip(self.config["sizes"], self.config["sizes"][1:]): self.p.extend([rng.normal(0, np.sqrt(2/a), (a, b)), np.zeros(b)]) self.m = [np.zeros_like(p) for p in self.p] self.v = [np.zeros_like(p) for p in self.p] self.last_grad = [np.zeros_like(p) for p in self.p] self.last_delta = [np.zeros_like(p) for p in self.p] self.steps = 0 def forward(self, x): a = x cache = [] for i, kind in enumerate(self.config["activations"]): z = a @ self.p[2*i] + self.p[2*i+1] result = activate(z, kind) cache.append((a, z, result, kind)) a = result return a, cache def predict(self, x): x = np.asarray(x, dtype=float) return self.forward(x / self.config["scale"])[0] * self.config["scale"] def gradients(self, x, y): prediction, cache = self.forward(x) delta = (prediction-y) / prediction.size grads = [None]*len(self.p) for i in reversed(range(len(cache))): before, z, a, kind = cache[i] dz = delta * derivative(z, a, kind) grads[2*i] = before.T @ dz grads[2*i+1] = dz.sum(axis=0) delta = dz @ self.p[2*i].T return grads def update(self, x, y): scale = self.config["scale"] gradients = self.gradients(np.asarray([x], float)/scale, np.asarray([y], float)/scale) if not all(np.isfinite(g).all() for g in gradients): raise ValueError("Numerisch instabiler Schritt. Lernrate oder Zahlenbereich verkleinern.") t = self.steps+1 new_p, new_m, new_v, deltas = [], [], [], [] with np.errstate(over="ignore", invalid="ignore"): for p, m, v, g in zip(self.p, self.m, self.v, gradients): m = .9*m+.1*g v = .999*v+.001*g*g delta = -self.config["learning_rate"]*(m/(1-.9**t))/(np.sqrt(v/(1-.999**t))+1e-8) new_p.append(p+delta) new_m.append(m); new_v.append(v); deltas.append(delta) if not all(np.isfinite(p).all() for p in new_p+new_m+new_v): raise ValueError("Numerisch instabiler Schritt; das bisherige Modell wurde beibehalten.") self.p, self.m, self.v = new_p, new_m, new_v self.last_grad, self.last_delta, self.steps = gradients, deltas, t class Lab: def __init__(self): self.create(DEFAULT, DEFAULT_TRAIN, DEFAULT_TEST, {"mode": "rounded", "tolerance": .01}) def create(self, config, train, test, criterion): config = config_valid(config) train, test = data_valid(train, config), data_valid(test, config, True) criterion = criterion_valid(criterion) if criterion["mode"] == "rounded" and any(v != round(v) for c in train+test for v in c["y"]): raise ValueError("Gerundete Antworten benötigen ganzzahlige Ziele. Für Dezimalziele Toleranz wählen.") model = Network(config) self.model, self.train, self.test, self.criterion = model, train, test, criterion self.revision, self.cursor = 1, 0 self.seen = set() self.history, self.versions = [], {} self.logged_components = 0 self.last_before = None self.new_revision() self.initial = self.metrics() def new_revision(self): self.versions[self.revision] = copy.deepcopy(dict(train=self.train, test=self.test, criterion=self.criterion)) def set_data(self, train, test, criterion): train = data_valid(train, self.model.config) test = data_valid(test, self.model.config, True) criterion = criterion_valid(criterion) if criterion["mode"] == "rounded" and any(v != round(v) for c in train+test for v in c["y"]): raise ValueError("Gerundete Antworten benötigen ganzzahlige Ziele.") self.train, self.test, self.criterion = train, test, criterion self.revision += 1 self.cursor = 0 self.last_before = None self.new_revision() def evaluate(self, cases): if not cases: return [] predictions = self.model.predict([c["x"] for c in cases]) result = [] for case, pred in zip(cases, predictions): error = np.abs(pred-np.asarray(case["y"])) if self.criterion["mode"] == "rounded": # Half away from zero, explicit and independent of Python's bankers rounding. rounded = np.copysign(np.floor(np.abs(pred)+.5), pred) correct = bool(np.all(rounded == np.asarray(case["y"]))) else: rounded = np.copysign(np.floor(np.abs(pred)+.5), pred) correct = bool(np.all(error < self.criterion["tolerance"])) result.append(dict(x=case["x"], y=case["y"], prediction=pred.tolist(), rounded=rounded.tolist(), error=error.tolist(), correct=correct, trained=tuple(case["x"]) in self.seen)) return result def metrics(self): train, test = self.evaluate(self.train), self.evaluate(self.test) def group(rows): errors = [e for row in rows for e in row["error"]] return dict(correct=sum(r["correct"] for r in rows), total=len(rows), accuracy=100*sum(r["correct"] for r in rows)/len(rows) if rows else None, mae=float(np.mean(errors)) if errors else None, max_error=max(errors) if errors else None) return dict(train=group(train), test=group(test)) def step(self, index=None): if index is None: index = self.cursor index = integer(index, "Trainingsaufgabe", 0, len(self.train)-1) # Preserve every step. Stop rather than silently dropping old measurements. additional = (len(self.train)+len(self.test))*self.model.config["sizes"][-1] if len(self.history) >= 10000 or self.logged_components + additional > 1000000: raise ValueError("Protokollgrenze erreicht. CSV exportieren und einen neuen Lauf beginnen.") before = self.evaluate(self.train)+self.evaluate(self.test) case = self.train[index] self.model.update(case["x"], case["y"]) self.seen.add(tuple(case["x"])) after = self.evaluate(self.train)+self.evaluate(self.test) self.last_before = before self.cursor = (index+1) % len(self.train) metrics = self.metrics() row = dict(step=self.model.steps, revision=self.revision, trained_index=index, trained_x=case["x"], trained_y=case["y"], loss=float(np.mean(np.square([e/self.model.config["scale"] for r in after[:len(self.train)] for e in r["error"]]))/2), train_accuracy=metrics["train"]["accuracy"], test_accuracy=metrics["test"]["accuracy"], train_max_error=metrics["train"]["max_error"], test_max_error=metrics["test"]["max_error"], forgotten=sum(a["correct"] and not b["correct"] for a,b in zip(before,after)), predictions=[r["prediction"] for r in after]) self.history.append(row) self.logged_components += additional return row def snapshot(self): train, test = self.evaluate(self.train), self.evaluate(self.test) if self.last_before is not None: for old, new in zip(self.last_before, train+test): new["before"] = old["prediction"] new["worsened"] = max(new["error"]) > max(old["error"])+1e-12 new["forgotten"] = old["correct"] and not new["correct"] history = self.history # Graph keeps all points up to 1000, then samples; CSV stays complete. stride = max(1, math.ceil(len(history)/1000)) graph = history[::stride] if history and (not graph or graph[-1] is not history[-1]): graph.append(history[-1]) return dict(config=self.model.config, parameters=sum(p.size for p in self.model.p), step=self.model.steps, cursor=self.cursor, revision=self.revision, criterion=self.criterion, train=train, test=test, metrics=self.metrics(), initial=self.initial, history=[{k:v for k,v in r.items() if k != "predictions"} for r in graph], last_steps=[{k:v for k,v in r.items() if k != "predictions"} for r in history[-12:]], formula=self.formula()) def formula(self): sizes=self.model.config["sizes"] if sizes not in ([2,1], [2,1,1]): return None p=self.model.p def f(v): return format(float(v),".7g") inner=f"{f(p[0][0,0])} · (x₁/{f(self.model.config['scale'])}) + {f(p[0][1,0])} · (x₂/{f(self.model.config['scale'])}) + {f(p[1][0])}" act=self.model.config["activations"][0] inner=f"{act}({inner})" if len(sizes)==3: inner=f"{self.model.config['activations'][1]}({f(p[2][0,0])} · {inner} + {f(p[3][0])})" return f"ŷ = {f(self.model.config['scale'])} · {inner}" def parameters(self, layer=0, neuron=0): layer=integer(layer,"Schicht",0,len(self.model.config["activations"])-1) neuron=integer(neuron,"Neuron",0,self.model.config["sizes"][layer+1]-1) rows=[] for i in range(self.model.config["sizes"][layer]): rows.append(dict(name=f"w{i+1}",value=float(self.model.p[2*layer][i,neuron]), gradient=float(self.model.last_grad[2*layer][i,neuron]), change=float(self.model.last_delta[2*layer][i,neuron]))) rows.append(dict(name="Bias",value=float(self.model.p[2*layer+1][neuron]), gradient=float(self.model.last_grad[2*layer+1][neuron]), change=float(self.model.last_delta[2*layer+1][neuron]))) return dict(layer=layer, neuron=neuron, rows=rows)