Add Tighter perceptron (#436)

Author: ishii-norimi

README.md

@@ -122,7 +122,7 @@ for (let i = 0; i < n; i++) {
 | task | model |
 | ---- | ----- |
 | clustering | (Soft / Kernel / Genetic / Weighted) k-means, k-means++, k-medois, k-medians, x-means, G-means, LBG, ISODATA, Fuzzy c-means, Possibilistic c-means, Agglomerative (complete linkage, single linkage, group average, Ward's, centroid, weighted average, median), DIANA, Monothetic, Mutual kNN, Mean shift, DBSCAN, OPTICS, HDBSCAN, DENCLUE, CLUES, PAM, CLARA, CLARANS, BIRCH, CURE, ROCK, C2P, PLSA, Latent dirichlet allocation, GMM, VBGMM, Affinity propagation, Spectral clustering, Mountain, SOM, GTM, (Growing) Neural gas, Growing cell structures, LVQ, ART, SVC, CAST, CHAMELEON, NMF, Autoencoder |
-| classification | (Fisher's) Linear discriminant, Quadratic discriminant, Mixture discriminant, Least squares, Ridge, (Complement / Negation / Universal-set / Selective) Naive Bayes (gaussian), AODE, (Fuzzy / Weighted) k-nearest neighbor, Radius neighbor, Nearest centroid, ENN, NNBCA, ADAMENN, DANN, IKNN, Decision tree, Random forest, Extra trees, GBDT, XGBoost, ALMA, (Aggressive) ROMMA, Online gradient descent, Passive aggressive, RLS, (Selective-sampling) Second order perceptron, AROW, NAROW, Confidence weighted, CELLIP, IELLIP, Normal herd, Stoptron, (Kernelized) Pegasos, MIRA, Forgetron, Projectron, Projectron++, Banditron, Ballseptron, BSGD, (Multinomial) Logistic regression, (Multinomial) Probit, SVM, Gaussian process, HMM, CRF, Bayesian Network, LVQ, (Average / Multiclass / Voted / Kernelized / Selective-sampling / Margin / Shifting / Budget) Perceptron, PAUM, RBP, ADALINE, MLP, LMNN |
+| classification | (Fisher's) Linear discriminant, Quadratic discriminant, Mixture discriminant, Least squares, Ridge, (Complement / Negation / Universal-set / Selective) Naive Bayes (gaussian), AODE, (Fuzzy / Weighted) k-nearest neighbor, Radius neighbor, Nearest centroid, ENN, NNBCA, ADAMENN, DANN, IKNN, Decision tree, Random forest, Extra trees, GBDT, XGBoost, ALMA, (Aggressive) ROMMA, Online gradient descent, Passive aggressive, RLS, (Selective-sampling) Second order perceptron, AROW, NAROW, Confidence weighted, CELLIP, IELLIP, Normal herd, Stoptron, (Kernelized) Pegasos, MIRA, Forgetron, Projectron, Projectron++, Banditron, Ballseptron, BSGD, (Multinomial) Logistic regression, (Multinomial) Probit, SVM, Gaussian process, HMM, CRF, Bayesian Network, LVQ, (Average / Multiclass / Voted / Kernelized / Selective-sampling / Margin / Shifting / Budget / Tighter) Perceptron, PAUM, RBP, ADALINE, MLP, LMNN |
 | semi-supervised classification | k-nearest neighbor, Radius neighbor, Label propagation, Label spreading, k-means, GMM, S3VM, Ladder network |
 | regression | Least squares, Ridge, Lasso, Elastic net, RLS, Bayesian linear, Poisson, Least absolute deviations, Huber, Tukey, Least trimmed squares, Least median squares, Lp norm linear, SMA, Deming, Segmented, LOWESS, spline, Gaussian process, Principal components, Partial least squares, Projection pursuit, Quantile regression, k-nearest neighbor, Radius neighbor, IDW, Nadaraya Watson, Priestley Chao, Gasser Muller, RBF Network, RVM, Decision tree, Random forest, Extra trees, GBDT, XGBoost, SVR, MLP, GMR, Isotonic, Ramer Douglas Peucker, Theil-Sen, Passing-Bablok, Repeated median |
 | interpolation | Nearest neighbor, IDW, (Spherical) Linear, Brahmagupta, Logarithmic, Cosine, (Inverse) Smoothstep, Cubic, (Centripetal) Catmull-Rom, Hermit, Polynomial, Lagrange, Trigonometric, Spline, RBF Network, Akima, Natural neighbor, Delaunay |

js/model_selector.js

@@ -183,6 +183,7 @@ const AIMethods = [
 				{ value: 'rbp', title: 'RBP' },
 				{ value: 'banditron', title: 'Banditron' },
 				{ value: 'ballseptron', title: 'Ballseptron' },
+				{ value: 'tighter_perceptron', title: 'Tighter Perceptron' },
 				{ value: 'bsgd', title: 'BSGD' },
 			],
 			Netrowk: [

js/view/tighter_perceptron.js

@@ -0,0 +1,39 @@
+import TighterPerceptron from '../../lib/model/tighter_perceptron.js'
+import EnsembleBinaryModel from '../../lib/model/ensemble_binary.js'
+import Controller from '../controller.js'
+
+export default function (platform) {
+	platform.setting.ml.usage = 'Click and add data point. Then, click "Step".'
+	const controller = new Controller(platform)
+	let model = null
+	const calc = cb => {
+		if (!model) {
+			model = new EnsembleBinaryModel(function () {
+				return new TighterPerceptron(beta.value, p.value, update.value)
+			}, method.value)
+			model.init(
+				platform.trainInput,
+				platform.trainOutput.map(v => v[0])
+			)
+		}
+		model.fit()
+		console.log(model)
+
+		const categories = model.predict(platform.testInput(3))
+		platform.testResult(categories)
+		cb && cb()
+	}
+
+	const method = controller.select(['oneone', 'onerest'])
+	const beta = controller.input.number({ label: ' beta ', min: 0, max: 100, step: 0.1, value: 0.1 })
+	const p = controller.input.number({ label: ' p ', min: 0, max: 1000, value: 10 })
+	const update = controller.select(['perceptron', 'mira', 'nobias'])
+	controller
+		.stepLoopButtons()
+		.init(() => {
+			model = null
+			platform.init()
+		})
+		.step(calc)
+		.epoch()
+}

lib/model/tighter_perceptron.js

@@ -0,0 +1,110 @@
+/**
+ * Tighter Budget Perceptron
+ */
+export default class TighterPerceptron {
+	// Online Learning: A Comprehensive Survey
+	// https://arxiv.org/abs/1802.02871
+	// Online (and Offline) on an Even Tighter Budget
+	// http://proceedings.mlr.press/r5/weston05a/weston05a.pdf
+	/**
+	 * @param {number} [beta=0] Margine
+	 * @param {number} [p=0] Cachs size
+	 * @param {'perceptron' | 'mira' | 'nobias'} [update=perceptron] Update rule
+	 */
+	constructor(beta = 0, p = 0, update = 'perceptron') {
+		this._beta = beta
+		this._p = p
+		this._C = 1
+
+		this._update = update
+		this._loss = (t, y) => {
+			return t === y ? 0 : 1
+		}
+	}
+
+	/**
+	 * Initialize this model.
+	 *
+	 * @param {Array<Array<number>>} train_x Training data
+	 * @param {Array<1 | -1>} train_y Target values
+	 */
+	init(train_x, train_y) {
+		this._x = train_x
+		this._y = train_y
+
+		if (this._update === 'mira') {
+			this._w = Array(this._x[0].length).fill(1)
+		} else {
+			this._w = Array(this._x[0].length).fill(0)
+		}
+		this._c = 0
+		this._a = []
+	}
+
+	/**
+	 * Fit model parameters.
+	 */
+	fit() {
+		for (let i = 0; i < this._x.length; i++) {
+			const pt = this._x[i].reduce((s, v, j) => s + v * this._w[j], this._c)
+			if (this._y[i] * pt > this._beta) {
+				continue
+			}
+			if (this._p > 0 && this._a.length >= this._p) {
+				let min_l = Infinity
+				let min_i = -1
+				for (let s = 0; s < this._a.length; s++) {
+					const a = this._a[s]
+					const ws = this._w.map((w, j) => w - a.a * this._y[a.i] * this._x[a.i][j])
+					const cs = this._c - a.a * this._y[a.i]
+					let v = 0
+					for (let k = 0; k < this._a.length; k++) {
+						let m = cs
+						for (let j = 0; j < this._w.length; j++) {
+							m += ws[j] * this._x[this._a[k].i][j]
+						}
+						v += this._loss(this._y[this._a[k].i], m <= 0 ? -1 : 1)
+					}
+					if (v < min_l) {
+						min_l = v
+						min_i = s
+					}
+				}
+				const ai = this._a[min_i]
+				for (let k = 0; k < this._w.length; k++) {
+					this._w[k] -= ai.a * this._y[ai.i] * this._x[ai.i][k]
+				}
+				this._c -= ai.a * this._y[ai.i]
+				this._a.splice(min_i, 1)
+			}
+			let alpha = 1
+			if (this._update === 'mira') {
+				const xx = this._x[i].reduce((s, v) => s + v ** 2, 1)
+				alpha = Math.min(1, Math.max(0, (-this._y[i] * pt) / xx))
+			} else if (this._update === 'nobias') {
+				const xx = this._x[i].reduce((s, v) => s + v ** 2, 1)
+				alpha = Math.min(this._C, Math.max(0, (1 - this._y[i] * pt) / xx))
+			}
+			this._a.push({ a: alpha, i })
+			for (let k = 0; k < this._w.length; k++) {
+				this._w[k] += alpha * this._y[i] * this._x[i][k]
+			}
+			this._c += alpha * this._y[i]
+		}
+	}
+
+	/**
+	 * Returns predicted values.
+	 *
+	 * @param {Array<Array<number>>} data Sample data
+	 * @returns {(1 | -1)[]} Predicted values
+	 */
+	predict(data) {
+		const p = []
+		for (let i = 0; i < data.length; i++) {
+			const m = data[i].reduce((s, v, j) => s + v * this._w[j], this._c)
+			p.push(m <= 0 ? -1 : 1)
+		}
+		return p
+	}
+}

tests/gui/view/tighter_perceptron.test.js

@@ -0,0 +1,57 @@
+import puppeteer from 'puppeteer'
+
+import { getPage } from '../helper/browser'
+
+describe('classification', () => {
+	/** @type {puppeteer.Page} */
+	let page
+	beforeEach(async () => {
+		page = await getPage()
+	}, 10000)
+
+	afterEach(async () => {
+		await page?.close()
+	})
+
+	test('initialize', async () => {
+		const taskSelectBox = await page.waitForSelector('#ml_selector dl:first-child dd:nth-child(5) select')
+		taskSelectBox.select('CF')
+		const modelSelectBox = await page.waitForSelector('#ml_selector .model_selection #mlDisp')
+		modelSelectBox.select('tighter_perceptron')
+		const methodMenu = await page.waitForSelector('#ml_selector #method_menu')
+		const buttons = await methodMenu.waitForSelector('.buttons')
+
+		const methods = await buttons.waitForSelector('select:nth-of-type(1)')
+		await expect((await methods.getProperty('value')).jsonValue()).resolves.toBe('oneone')
+		const beta = await buttons.waitForSelector('input:nth-of-type(1)')
+		await expect((await beta.getProperty('value')).jsonValue()).resolves.toBe('0.1')
+		const p = await buttons.waitForSelector('input:nth-of-type(2)')
+		await expect((await p.getProperty('value')).jsonValue()).resolves.toBe('10')
+		const update = await buttons.waitForSelector('select:nth-of-type(2)')
+		await expect((await update.getProperty('value')).jsonValue()).resolves.toBe('perceptron')
+		const epoch = await buttons.waitForSelector('[name=epoch]')
+		await expect(epoch.evaluate(el => el.textContent)).resolves.toBe('0')
+	}, 10000)
+
+	test('learn', async () => {
+		const taskSelectBox = await page.waitForSelector('#ml_selector dl:first-child dd:nth-child(5) select')
+		taskSelectBox.select('CF')
+		const modelSelectBox = await page.waitForSelector('#ml_selector .model_selection #mlDisp')
+		modelSelectBox.select('tighter_perceptron')
+		const methodMenu = await page.waitForSelector('#ml_selector #method_menu')
+		const buttons = await methodMenu.waitForSelector('.buttons')
+
+		const epoch = await buttons.waitForSelector('[name=epoch]')
+		await expect(epoch.evaluate(el => el.textContent)).resolves.toBe('0')
+		const methodFooter = await page.waitForSelector('#method_footer')
+		await expect(methodFooter.evaluate(el => el.textContent)).resolves.toBe('')
+
+		const initButton = await buttons.waitForSelector('input[value=Initialize]')
+		await initButton.evaluate(el => el.click())
+		const stepButton = await buttons.waitForSelector('input[value=Step]:enabled')
+		await stepButton.evaluate(el => el.click())
+
+		await expect(epoch.evaluate(el => el.textContent)).resolves.toBe('1')
+		await expect(methodFooter.evaluate(el => el.textContent)).resolves.toMatch(/^Accuracy:[0-9.]+$/)
+	}, 10000)
+})

tests/lib/model/tighter_perceptron.test.js

@@ -0,0 +1,26 @@
+import { jest } from '@jest/globals'
+jest.retryTimes(3)
+
+import Matrix from '../../../lib/util/matrix.js'
+import TighterPerceptron from '../../../lib/model/tighter_perceptron.js'
+
+import { accuracy } from '../../../lib/evaluate/classification.js'
+
+describe('classification', () => {
+	test.each([undefined, 'perceptron', 'mira', 'nobias'])('update %s', update => {
+		const model = new TighterPerceptron(1, 10, update)
+		const x = Matrix.concat(Matrix.randn(50, 2, 0, 0.2), Matrix.randn(50, 2, 5, 0.2)).toArray()
+		x[50] = [0.1, 0.1]
+		const t = []
+		for (let i = 0; i < x.length; i++) {
+			t[i] = Math.floor(i / 50) * 2 - 1
+		}
+		model.init(x, t)
+		for (let i = 0; i < 10; i++) {
+			model.fit()
+		}
+		const y = model.predict(x)
+		const acc = accuracy(y, t)
+		expect(acc).toBeGreaterThan(0.8)
+	})
+})