diff --git a/client/__pycache__/settings.cpython-36.pyc b/client/__pycache__/settings.cpython-36.pyc new file mode 100644 index 0000000..5b671a0 Binary files /dev/null and b/client/__pycache__/settings.cpython-36.pyc differ diff --git a/client/__pycache__/settings_local.cpython-36.pyc b/client/__pycache__/settings_local.cpython-36.pyc new file mode 100644 index 0000000..4017133 Binary files /dev/null and b/client/__pycache__/settings_local.cpython-36.pyc differ diff --git a/client/benchmarks/__pycache__/__init__.cpython-36.pyc b/client/benchmarks/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..f6970ef Binary files /dev/null and b/client/benchmarks/__pycache__/__init__.cpython-36.pyc differ diff --git a/client/benchmarks/__pycache__/pgbench.cpython-36.pyc b/client/benchmarks/__pycache__/pgbench.cpython-36.pyc new file mode 100644 index 0000000..8ace00b Binary files /dev/null and b/client/benchmarks/__pycache__/pgbench.cpython-36.pyc differ diff --git a/client/benchmarks/__pycache__/runner.cpython-36.pyc b/client/benchmarks/__pycache__/runner.cpython-36.pyc new file mode 100644 index 0000000..34709d5 Binary files /dev/null and b/client/benchmarks/__pycache__/runner.cpython-36.pyc differ diff --git a/client/benchmarks/pgbench.py b/client/benchmarks/pgbench.py index 3cf2719..4ff5bf7 100644 --- a/client/benchmarks/pgbench.py +++ b/client/benchmarks/pgbench.py @@ -214,7 +214,7 @@ def run_tests(self, csv_queue): # derive configuration for the CPU count / RAM size configs = PgBench._configure(cpu_count(), available_ram()) - results = {'ro': {}, 'rw': {}} + results = {'ro': {}, 'rw': {}} #ro:read only rw:read-write j = 0 for config in configs: scale = config['scale'] diff --git a/client/benchmarks/runner.py b/client/benchmarks/runner.py index 8544fac..4b7a73c 100644 --- a/client/benchmarks/runner.py +++ b/client/benchmarks/runner.py @@ -83,7 +83,8 @@ def _run_config(self, config_name): # expand the attribute names bench = bench(**config['config']) - self._cluster.start(config=config['postgres']) + self._cluster.start( + config=config['postgres']) # obtain the database name pgperffarm-db # start collector(s) of additional info self._collector.start() @@ -116,16 +117,16 @@ def _run_config(self, config_name): uname = check_output(['uname', '-a']) r['meta'] = { - 'benchmark': config['benchmark'], - 'date': strftime("%Y-%m-%d %H:%M:%S.000000+00", gmtime()), - 'name': config_name, - 'uname': uname, + 'benchmark': config['benchmark'], + 'date': strftime("%Y-%m-%d %H:%M:%S.000000+00", gmtime()), + 'name': config_name, + 'uname': uname, } r['postgres'] = { - 'branch': config['branch'], - 'commit': config['commit'], - 'settings': config['postgres'], + 'branch': config['branch'], + 'commit': config['commit'], + 'settings': config['postgres'], } with open('%s/results.json' % self._output, 'w') as f: @@ -134,23 +135,23 @@ def _run_config(self, config_name): try: self._upload_results(r) except Exception as e: - print (e) + print(e) def _upload_results(self, results): - postdata = results - post = [] + postdata = results + post = [] post.append(postdata) headers = {'Content-Type': 'application/json; charset=utf-8', 'Authorization': self._secret} + # r = requests.post(self._url+"upload/".encode('utf-8'), data=json.dumps(post).encode('utf-8'), headers=headers) r = requests.post(self._url.encode('utf-8'), data=json.dumps(post).encode('utf-8'), headers=headers) - def run(self): 'run all the configured benchmarks' # Removing the existing directory - + try: os.mkdir(self._output) except OSError as e: diff --git a/client/collectors/__pycache__/__init__.cpython-36.pyc b/client/collectors/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..4667920 Binary files /dev/null and b/client/collectors/__pycache__/__init__.cpython-36.pyc differ diff --git a/client/collectors/__pycache__/collectd.cpython-36.pyc b/client/collectors/__pycache__/collectd.cpython-36.pyc new file mode 100644 index 0000000..8edc1bd Binary files /dev/null and b/client/collectors/__pycache__/collectd.cpython-36.pyc differ diff --git a/client/collectors/__pycache__/collector.cpython-36.pyc b/client/collectors/__pycache__/collector.cpython-36.pyc new file mode 100644 index 0000000..1253f56 Binary files /dev/null and b/client/collectors/__pycache__/collector.cpython-36.pyc differ diff --git a/client/collectors/__pycache__/linux.cpython-36.pyc b/client/collectors/__pycache__/linux.cpython-36.pyc new file mode 100644 index 0000000..f096a39 Binary files /dev/null and b/client/collectors/__pycache__/linux.cpython-36.pyc differ diff --git a/client/collectors/__pycache__/postgres.cpython-36.pyc b/client/collectors/__pycache__/postgres.cpython-36.pyc new file mode 100644 index 0000000..58df105 Binary files /dev/null and b/client/collectors/__pycache__/postgres.cpython-36.pyc differ diff --git a/client/collectors/collectd.py b/client/collectors/collectd.py index 1cbe3b8..143983e 100644 --- a/client/collectors/collectd.py +++ b/client/collectors/collectd.py @@ -3,8 +3,8 @@ from utils.logging import log from utils.misc import run_cmd -COLLECTD_CONFIG = '/tmp/.collectd.conf' -COLLECTD_PIDFILE = '/tmp/.collectd.pid' +COLLECTD_CONFIG = '/raid/.collectd.conf' +COLLECTD_PIDFILE = '/raid/.collectd.pid' class CollectdCollector(object): diff --git a/client/perffarm-client.py b/client/perffarm-client.py index a0e4c7d..0edd461 100644 --- a/client/perffarm-client.py +++ b/client/perffarm-client.py @@ -19,8 +19,9 @@ from settings_local import * from settings import * -API_URL = 'http://127.0.0.1:8000/' -MACHINE_SECRET = '610f79063e62e6ad09460ac2c4e66da0386dc89b' +# API_URL = 'http://127.0.0.1:8000/' +API_URL = 'http://127.0.0.1:8000/upload/' +MACHINE_SECRET = 'e984c3017cd1a0dff0ef9f0c394a5c285e421411' #This machine secrete is specific to my environment if __name__ == '__main__': with FileLock('.lock') as lock: diff --git a/client/settings.py b/client/settings.py index 0a4f31f..5508624 100644 --- a/client/settings.py +++ b/client/settings.py @@ -2,11 +2,11 @@ import sys # global configuration -GIT_URL = 'https://github.com/postgres/postgres.git' -REPOSITORY_PATH = '/tmp/git-postgres' -BUILD_PATH = '/Users/chenzhang/anaconda3' +GIT_URL = 'https://gitee.com/purpleyu/postgres.git' #changed to a local repo +REPOSITORY_PATH = '/raid/git-postgres/postgres' +BUILD_PATH = '/usr/lib/postgresql/11' BIN_PATH = os.path.join(BUILD_PATH, 'bin') -DATADIR_PATH = '/tmp/data-postgres' +DATADIR_PATH = '/raid/data-postgres/postgres' POSTGRES_CONFIG = { 'shared_buffers': '1GB', @@ -22,9 +22,9 @@ 'checkpoint_completion_target': '0.9', } -DATABASE_NAME = 'postgres' # This name needs to be the same as rest_api settings_local.py database NAME +DATABASE_NAME = 'pgperffarm-db' # This name needs to be the same as rest_api settings_local.py database NAME -OUTPUT_DIR = '/tmp/perf-output' +OUTPUT_DIR = '/raid/perf-output' # configuration for PgBench # runs - number of repetitions (including test for all client counts) diff --git a/client/utils/__pycache__/__init__.cpython-36.pyc b/client/utils/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..f6cb89d Binary files /dev/null and b/client/utils/__pycache__/__init__.cpython-36.pyc differ diff --git a/client/utils/__pycache__/cluster.cpython-36.pyc b/client/utils/__pycache__/cluster.cpython-36.pyc new file mode 100644 index 0000000..1eb18e5 Binary files /dev/null and b/client/utils/__pycache__/cluster.cpython-36.pyc differ diff --git a/client/utils/__pycache__/git.cpython-36.pyc b/client/utils/__pycache__/git.cpython-36.pyc new file mode 100644 index 0000000..92e85c8 Binary files /dev/null and b/client/utils/__pycache__/git.cpython-36.pyc differ diff --git a/client/utils/__pycache__/locking.cpython-36.pyc b/client/utils/__pycache__/locking.cpython-36.pyc new file mode 100644 index 0000000..00660c4 Binary files /dev/null and b/client/utils/__pycache__/locking.cpython-36.pyc differ diff --git a/client/utils/__pycache__/logging.cpython-36.pyc b/client/utils/__pycache__/logging.cpython-36.pyc new file mode 100644 index 0000000..ffc6c98 Binary files /dev/null and b/client/utils/__pycache__/logging.cpython-36.pyc differ diff --git a/client/utils/__pycache__/misc.cpython-36.pyc b/client/utils/__pycache__/misc.cpython-36.pyc new file mode 100644 index 0000000..65b997d Binary files /dev/null and b/client/utils/__pycache__/misc.cpython-36.pyc differ diff --git a/rest_api/apps/machines/__pycache__/__init__.cpython-36.pyc b/rest_api/apps/machines/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..3ced7f3 Binary files /dev/null and b/rest_api/apps/machines/__pycache__/__init__.cpython-36.pyc differ diff --git a/rest_api/apps/machines/__pycache__/admin.cpython-36.pyc b/rest_api/apps/machines/__pycache__/admin.cpython-36.pyc new file mode 100644 index 0000000..6f56006 Binary files /dev/null and b/rest_api/apps/machines/__pycache__/admin.cpython-36.pyc differ diff --git a/rest_api/apps/machines/__pycache__/models.cpython-36.pyc b/rest_api/apps/machines/__pycache__/models.cpython-36.pyc new file mode 100644 index 0000000..c38faf9 Binary files /dev/null and b/rest_api/apps/machines/__pycache__/models.cpython-36.pyc differ diff --git a/rest_api/apps/machines/__pycache__/permissions.cpython-36.pyc b/rest_api/apps/machines/__pycache__/permissions.cpython-36.pyc new file mode 100644 index 0000000..2924fcb Binary files /dev/null and b/rest_api/apps/machines/__pycache__/permissions.cpython-36.pyc differ diff --git a/rest_api/apps/machines/__pycache__/serializers.cpython-36.pyc b/rest_api/apps/machines/__pycache__/serializers.cpython-36.pyc new file mode 100644 index 0000000..2e6dbf9 Binary files /dev/null and b/rest_api/apps/machines/__pycache__/serializers.cpython-36.pyc differ diff --git a/rest_api/apps/machines/__pycache__/urls.cpython-36.pyc b/rest_api/apps/machines/__pycache__/urls.cpython-36.pyc new file mode 100644 index 0000000..5231aa9 Binary files /dev/null and b/rest_api/apps/machines/__pycache__/urls.cpython-36.pyc differ diff --git a/rest_api/apps/machines/__pycache__/views.cpython-36.pyc b/rest_api/apps/machines/__pycache__/views.cpython-36.pyc new file mode 100644 index 0000000..601b80f Binary files /dev/null and b/rest_api/apps/machines/__pycache__/views.cpython-36.pyc differ diff --git a/rest_api/apps/machines/migrations/0001_initial.py b/rest_api/apps/machines/migrations/0001_initial.py new file mode 100644 index 0000000..d17ad8c --- /dev/null +++ b/rest_api/apps/machines/migrations/0001_initial.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.17 on 2020-02-03 17:22 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Alias', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=32, unique=True, verbose_name='alias name')), + ('is_used', models.BooleanField(default=False, verbose_name='is_used')), + ('add_time', models.DateTimeField(auto_now_add=True)), + ], + ), + migrations.CreateModel( + name='Machine', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('add_time', models.DateTimeField(auto_now_add=True)), + ('alias', models.CharField(blank=True, default='', max_length=100)), + ('machine_secret', models.CharField(blank=True, default='', max_length=32, verbose_name='machine secret')), + ('sn', models.CharField(blank=True, default='', max_length=16)), + ('os_name', models.CharField(blank=True, default='', max_length=100)), + ('os_version', models.CharField(blank=True, default='', max_length=100)), + ('comp_name', models.CharField(blank=True, default='', max_length=100)), + ('comp_version', models.CharField(blank=True, default='', max_length=100)), + ('state', models.CharField(choices=[('A', 'Active'), ('I', 'Inactive')], default='A', max_length=10)), + ('owner_email', models.EmailField(max_length=256, verbose_name='email')), + ('owner_username', models.CharField(blank=True, default='', max_length=100)), + ('owner_id', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='machines', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ('add_time',), + }, + ), + ] diff --git a/rest_api/apps/machines/migrations/0002_auto_20200213_1523.py b/rest_api/apps/machines/migrations/0002_auto_20200213_1523.py new file mode 100644 index 0000000..813015f --- /dev/null +++ b/rest_api/apps/machines/migrations/0002_auto_20200213_1523.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.17 on 2020-02-13 15:23 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('machines', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='machine', + name='machine_secret', + field=models.CharField(blank=True, default='', max_length=100, verbose_name='machine secret'), + ), + ] diff --git a/rest_api/apps/machines/migrations/__init__.py b/rest_api/apps/machines/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rest_api/apps/machines/migrations/__pycache__/0001_initial.cpython-36.pyc b/rest_api/apps/machines/migrations/__pycache__/0001_initial.cpython-36.pyc new file mode 100644 index 0000000..617499c Binary files /dev/null and b/rest_api/apps/machines/migrations/__pycache__/0001_initial.cpython-36.pyc differ diff --git a/rest_api/apps/machines/migrations/__pycache__/0002_auto_20200213_1523.cpython-36.pyc b/rest_api/apps/machines/migrations/__pycache__/0002_auto_20200213_1523.cpython-36.pyc new file mode 100644 index 0000000..4dbb598 Binary files /dev/null and b/rest_api/apps/machines/migrations/__pycache__/0002_auto_20200213_1523.cpython-36.pyc differ diff --git a/rest_api/apps/machines/migrations/__pycache__/__init__.cpython-36.pyc b/rest_api/apps/machines/migrations/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..0851e01 Binary files /dev/null and b/rest_api/apps/machines/migrations/__pycache__/__init__.cpython-36.pyc differ diff --git a/rest_api/apps/machines/models.py b/rest_api/apps/machines/models.py index c9bc731..d45f10f 100644 --- a/rest_api/apps/machines/models.py +++ b/rest_api/apps/machines/models.py @@ -29,7 +29,7 @@ class Machine(models.Model): add_time = models.DateTimeField(auto_now_add=True) alias = models.CharField(max_length=100, blank=True, default='') - machine_secret = models.CharField(max_length=32, blank=True, default='', verbose_name="machine secret") + machine_secret = models.CharField(max_length=100, blank=True, default='', verbose_name="machine secret") sn = models.CharField(max_length=16, blank=True, default='') os_name = models.CharField(max_length=100, blank=True, default='') os_version = models.CharField(max_length=100, blank=True, default='') diff --git a/rest_api/apps/records/__pycache__/__init__.cpython-36.pyc b/rest_api/apps/records/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..29cb07a Binary files /dev/null and b/rest_api/apps/records/__pycache__/__init__.cpython-36.pyc differ diff --git a/rest_api/apps/records/__pycache__/admin.cpython-36.pyc b/rest_api/apps/records/__pycache__/admin.cpython-36.pyc new file mode 100644 index 0000000..a9137f9 Binary files /dev/null and b/rest_api/apps/records/__pycache__/admin.cpython-36.pyc differ diff --git a/rest_api/apps/records/__pycache__/exception.cpython-36.pyc b/rest_api/apps/records/__pycache__/exception.cpython-36.pyc new file mode 100644 index 0000000..6380ffe Binary files /dev/null and b/rest_api/apps/records/__pycache__/exception.cpython-36.pyc differ diff --git a/rest_api/apps/records/__pycache__/filters.cpython-36.pyc b/rest_api/apps/records/__pycache__/filters.cpython-36.pyc new file mode 100644 index 0000000..050a956 Binary files /dev/null and b/rest_api/apps/records/__pycache__/filters.cpython-36.pyc differ diff --git a/rest_api/apps/records/__pycache__/models.cpython-36.pyc b/rest_api/apps/records/__pycache__/models.cpython-36.pyc new file mode 100644 index 0000000..629538b Binary files /dev/null and b/rest_api/apps/records/__pycache__/models.cpython-36.pyc differ diff --git a/rest_api/apps/records/__pycache__/serializers.cpython-36.pyc b/rest_api/apps/records/__pycache__/serializers.cpython-36.pyc new file mode 100644 index 0000000..1d14c40 Binary files /dev/null and b/rest_api/apps/records/__pycache__/serializers.cpython-36.pyc differ diff --git a/rest_api/apps/records/__pycache__/urls.cpython-36.pyc b/rest_api/apps/records/__pycache__/urls.cpython-36.pyc new file mode 100644 index 0000000..9f674c5 Binary files /dev/null and b/rest_api/apps/records/__pycache__/urls.cpython-36.pyc differ diff --git a/rest_api/apps/records/__pycache__/views.cpython-36.pyc b/rest_api/apps/records/__pycache__/views.cpython-36.pyc new file mode 100644 index 0000000..e2e6d7c Binary files /dev/null and b/rest_api/apps/records/__pycache__/views.cpython-36.pyc differ diff --git a/rest_api/apps/records/migrations/0001_initial.py b/rest_api/apps/records/migrations/0001_initial.py new file mode 100644 index 0000000..05db8e8 --- /dev/null +++ b/rest_api/apps/records/migrations/0001_initial.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.17 on 2020-02-03 17:22 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('machines', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='LinuxInfo', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('mounts', models.TextField(default='null', help_text='mounts', verbose_name='mounts')), + ('cpuinfo', models.TextField(default='null', help_text='cpuinfo', verbose_name='cpuinfo')), + ('sysctl', models.TextField(default='null', help_text='sysctl', verbose_name='sysctl')), + ('meminfo', models.TextField(default='null', help_text='meminfo', verbose_name='meminfo')), + ], + options={ + 'verbose_name': 'linux info', + 'verbose_name_plural': 'linux info', + }, + ), + migrations.CreateModel( + name='MetaInfo', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date', models.DateTimeField(help_text='date', verbose_name='date')), + ('uname', models.TextField(help_text='uname', verbose_name='uname')), + ('benchmark', models.TextField(help_text='benchmark', verbose_name='benchmark')), + ('name', models.TextField(help_text='name', verbose_name='name')), + ], + options={ + 'verbose_name': 'meta info', + 'verbose_name_plural': 'meta info', + }, + ), + migrations.CreateModel( + name='PGInfo', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('checkpoint_timeout', models.CharField(help_text='checkpoint_timeout', max_length=32, verbose_name='checkpoint_timeout')), + ('log_temp_files', models.IntegerField(help_text='log_temp_files', verbose_name='log_temp_files')), + ('work_mem', models.CharField(help_text='work_mem', max_length=32, verbose_name='work_mem')), + ('log_line_prefix', models.CharField(help_text='checkpoint_timeout', max_length=64, verbose_name='checkpoint_timeout')), + ('shared_buffers', models.CharField(help_text='shared_buffers', max_length=32, verbose_name='shared_buffers')), + ('log_autovacuum_min_duration', models.IntegerField(help_text='log_autovacuum_min_duration', verbose_name='log_autovacuum_min_duration')), + ('checkpoint_completion_target', models.DecimalField(decimal_places=4, help_text='checkpoint_completion_target', max_digits=8, verbose_name='checkpoint_completion_target')), + ('maintenance_work_mem', models.CharField(help_text='maintenance_work_mem', max_length=32, verbose_name='maintenance_work_mem')), + ('log_checkpoints', models.IntegerField(choices=[(1, 'on'), (2, 'off')], help_text='log_checkpoints', verbose_name='log_checkpoints')), + ('max_wal_size', models.CharField(help_text='max_wal_size', max_length=32, verbose_name='max_wal_size')), + ('min_wal_size', models.CharField(help_text='min_wal_size', max_length=32, verbose_name='min_wal_size')), + ], + options={ + 'verbose_name': 'pg info', + 'verbose_name_plural': 'pg info', + }, + ), + migrations.CreateModel( + name='TestBranch', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('branch_name', models.CharField(help_text='branch name', max_length=128, unique=True, verbose_name='branch name')), + ('branch_order', models.IntegerField(default=5, help_text='order in all the branch', verbose_name='branch order')), + ('is_show', models.BooleanField(default=True, help_text='branch isshow', verbose_name='branch is shown')), + ('is_accept', models.BooleanField(default=True, help_text='branch accepts new reports', verbose_name='branch accepts new reports')), + ('add_time', models.DateTimeField(default=django.utils.timezone.now, help_text='branch added time', verbose_name='branch added time')), + ], + options={ + 'verbose_name': 'test branch', + 'verbose_name_plural': 'test branch', + }, + ), + migrations.CreateModel( + name='TestCategory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('cate_name', models.CharField(help_text='cate name', max_length=64, verbose_name='cate name')), + ('cate_sn', models.CharField(help_text='cate sn', max_length=32, unique=True, verbose_name='cate sn')), + ('cate_order', models.IntegerField(help_text='order in the current level', verbose_name='cate order')), + ('add_time', models.DateTimeField(default=django.utils.timezone.now, help_text='category added time', verbose_name='add time')), + ], + options={ + 'verbose_name': 'tests category', + 'verbose_name_plural': 'tests category', + }, + ), + migrations.CreateModel( + name='TestDataSet', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('clients', models.IntegerField(help_text='clients of the test dataset', verbose_name='clients')), + ('scale', models.IntegerField(help_text='scale of the test dataset', verbose_name='scale')), + ('std', models.DecimalField(decimal_places=8, help_text='std of the test dataset', max_digits=18, verbose_name='std')), + ('metric', models.DecimalField(decimal_places=8, help_text='metric of the test dataset', max_digits=18, verbose_name='metric')), + ('median', models.DecimalField(decimal_places=8, help_text='median of the test dataset', max_digits=18, verbose_name='median')), + ('status', models.IntegerField(choices=[(-1, 'none'), (1, 'improved'), (2, 'quo'), (3, 'regressive')], help_text='status of this dataset', verbose_name='status')), + ('percentage', models.DecimalField(decimal_places=4, help_text='percentage compared to previous dataset', max_digits=8, verbose_name='percentage')), + ('add_time', models.DateTimeField(default=django.utils.timezone.now, verbose_name='test dataset time')), + ('prev', models.ForeignKey(blank=True, help_text='previous test dataset id', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='prev1', to='records.TestDataSet', verbose_name='previous test dataset id')), + ('test_cate', models.ForeignKey(help_text='test cate id', on_delete=django.db.models.deletion.CASCADE, to='records.TestCategory', verbose_name='test cate id')), + ], + options={ + 'verbose_name': 'test dataset', + 'verbose_name_plural': 'test dataset', + }, + ), + migrations.CreateModel( + name='TestRecord', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('test_desc', models.TextField(help_text='test desc', verbose_name='test desc')), + ('meta_time', models.DateTimeField(default=django.utils.timezone.now, verbose_name='meta time')), + ('hash', models.CharField(default='', help_text='record hash', max_length=128, unique=True, verbose_name='record hash')), + ('uuid', models.CharField(default='', help_text='record uuid', max_length=64, unique=True, verbose_name='record uuid')), + ('commit', models.CharField(help_text='record commit', max_length=64, verbose_name='record commit')), + ('add_time', models.DateTimeField(default=django.utils.timezone.now, verbose_name='test added time')), + ('branch', models.ForeignKey(help_text='pg branch', on_delete=django.db.models.deletion.CASCADE, to='records.TestBranch', verbose_name='pg branch')), + ('linux_info', models.ForeignKey(help_text='linux info', on_delete=django.db.models.deletion.CASCADE, to='records.LinuxInfo', verbose_name='linux info')), + ('meta_info', models.ForeignKey(help_text='meta info', on_delete=django.db.models.deletion.CASCADE, to='records.MetaInfo', verbose_name='meta info')), + ('pg_info', models.ForeignKey(help_text='pg info', on_delete=django.db.models.deletion.CASCADE, to='records.PGInfo', verbose_name='pg info')), + ('test_machine', models.ForeignKey(help_text='person who add this test item', on_delete=django.db.models.deletion.CASCADE, to='machines.Machine', verbose_name='test owner')), + ], + options={ + 'verbose_name': 'tests', + 'verbose_name_plural': 'tests', + }, + ), + migrations.CreateModel( + name='TestResult', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('latency', models.IntegerField(help_text='latency of the test result', verbose_name='latency')), + ('scale', models.IntegerField(help_text='scale of the test result', verbose_name='scale')), + ('end', models.DecimalField(decimal_places=12, help_text='endtime of the test result', max_digits=32, verbose_name='end')), + ('clients', models.IntegerField(help_text='clients of the test result', verbose_name='clients')), + ('start', models.DecimalField(decimal_places=12, help_text='starttime of the test result', max_digits=32, verbose_name='start')), + ('tps', models.DecimalField(decimal_places=6, default=0, help_text='tps of the test result', max_digits=18, verbose_name='tps')), + ('run', models.IntegerField(help_text='run number', verbose_name='run')), + ('threads', models.IntegerField(help_text='threads of the test result', verbose_name='threads')), + ('mode', models.IntegerField(choices=[(1, 'simple'), (2, 'other'), (-1, 'test')], help_text='test mode', verbose_name='mode')), + ('add_time', models.DateTimeField(default=django.utils.timezone.now, verbose_name='test result added time')), + ('test_dataset', models.ForeignKey(help_text='test dataset id', on_delete=django.db.models.deletion.CASCADE, to='records.TestDataSet', verbose_name='test dataset id')), + ], + options={ + 'verbose_name': 'test result', + 'verbose_name_plural': 'test result', + }, + ), + migrations.AddField( + model_name='testdataset', + name='test_record', + field=models.ForeignKey(help_text='test record id', on_delete=django.db.models.deletion.CASCADE, to='records.TestRecord', verbose_name='test record id'), + ), + ] diff --git a/rest_api/apps/records/migrations/0002_auto_20200215_1749.py b/rest_api/apps/records/migrations/0002_auto_20200215_1749.py new file mode 100644 index 0000000..65a4849 --- /dev/null +++ b/rest_api/apps/records/migrations/0002_auto_20200215_1749.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.17 on 2020-02-15 17:49 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('records', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='testrecord', + name='commit', + field=models.CharField(help_text='record commit', max_length=100, verbose_name='record commit'), + ), + ] diff --git a/rest_api/apps/records/migrations/__init__.py b/rest_api/apps/records/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rest_api/apps/records/migrations/__pycache__/0001_initial.cpython-36.pyc b/rest_api/apps/records/migrations/__pycache__/0001_initial.cpython-36.pyc new file mode 100644 index 0000000..6e49cd7 Binary files /dev/null and b/rest_api/apps/records/migrations/__pycache__/0001_initial.cpython-36.pyc differ diff --git a/rest_api/apps/records/migrations/__pycache__/0002_auto_20200215_1749.cpython-36.pyc b/rest_api/apps/records/migrations/__pycache__/0002_auto_20200215_1749.cpython-36.pyc new file mode 100644 index 0000000..785d507 Binary files /dev/null and b/rest_api/apps/records/migrations/__pycache__/0002_auto_20200215_1749.cpython-36.pyc differ diff --git a/rest_api/apps/records/migrations/__pycache__/__init__.cpython-36.pyc b/rest_api/apps/records/migrations/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..f4d7bb7 Binary files /dev/null and b/rest_api/apps/records/migrations/__pycache__/__init__.cpython-36.pyc differ diff --git a/rest_api/apps/records/models.py b/rest_api/apps/records/models.py index 64b2e2f..c01f27f 100644 --- a/rest_api/apps/records/models.py +++ b/rest_api/apps/records/models.py @@ -10,10 +10,11 @@ class TestBranch(models.Model): """ test branch """ - branch_name = models.CharField(max_length=128, unique=True,verbose_name="branch name", help_text="branch name") - branch_order = models.IntegerField(default=5,verbose_name="branch order", help_text="order in all the branch") - is_show = models.BooleanField(verbose_name="branch is shown", default=True, help_text="branch isshow") - is_accept = models.BooleanField(verbose_name="branch accepts new reports", default=True, help_text="branch accepts new reports") + branch_name = models.CharField(max_length=128, unique=True, verbose_name="branch name", help_text="branch name") + branch_order = models.IntegerField(default=5, verbose_name="branch order", help_text="order in all the branch") + is_show = models.BooleanField(verbose_name="branch is shown", default=True, help_text="branch is show") + is_accept = models.BooleanField(verbose_name="branch accepts new reports", default=True, + help_text="branch accepts new reports") add_time = models.DateTimeField(default=timezone.now, verbose_name="branch added time", help_text="branch added time") @@ -43,23 +44,27 @@ def __str__(self): class PGInfo(models.Model): - - checkpoint_timeout = models.CharField(max_length=32, verbose_name="checkpoint_timeout", help_text="checkpoint_timeout") + checkpoint_timeout = models.CharField(max_length=32, verbose_name="checkpoint_timeout", + help_text="checkpoint_timeout") log_temp_files = models.IntegerField(verbose_name="log_temp_files", help_text="log_temp_files") work_mem = models.CharField(max_length=32, verbose_name="work_mem", help_text="work_mem") - log_line_prefix = models.CharField(max_length=64,verbose_name="checkpoint_timeout", help_text="checkpoint_timeout") + log_line_prefix = models.CharField(max_length=64, verbose_name="checkpoint_timeout", help_text="checkpoint_timeout") shared_buffers = models.CharField(max_length=32, verbose_name="shared_buffers", help_text="shared_buffers") - log_autovacuum_min_duration =models.IntegerField(verbose_name="log_autovacuum_min_duration", help_text="log_autovacuum_min_duration") - + log_autovacuum_min_duration = models.IntegerField(verbose_name="log_autovacuum_min_duration", + help_text="log_autovacuum_min_duration") - checkpoint_completion_target = models.DecimalField(max_digits=8, decimal_places=4,verbose_name="checkpoint_completion_target", help_text="checkpoint_completion_target") - maintenance_work_mem = models.CharField(max_length=32, verbose_name="maintenance_work_mem", help_text="maintenance_work_mem") + checkpoint_completion_target = models.DecimalField(max_digits=8, decimal_places=4, + verbose_name="checkpoint_completion_target", + help_text="checkpoint_completion_target") + maintenance_work_mem = models.CharField(max_length=32, verbose_name="maintenance_work_mem", + help_text="maintenance_work_mem") SWITCH_CHOICE = ( (1, 'on'), (2, 'off') ) - log_checkpoints = models.IntegerField(choices=SWITCH_CHOICE,verbose_name="log_checkpoints", help_text="log_checkpoints") + log_checkpoints = models.IntegerField(choices=SWITCH_CHOICE, verbose_name="log_checkpoints", + help_text="log_checkpoints") max_wal_size = models.CharField(max_length=32, verbose_name="max_wal_size", help_text="max_wal_size") min_wal_size = models.CharField(max_length=32, verbose_name="min_wal_size", help_text="min_wal_size") @@ -117,7 +122,7 @@ class TestRecord(models.Model): hash = models.CharField(unique=True, default='', max_length=128, verbose_name="record hash", help_text="record hash") uuid = models.CharField(unique=True, default='', max_length=64, verbose_name="record uuid", help_text="record uuid") - commit = models.CharField(max_length=64, verbose_name="record commit", help_text="record commit") + commit = models.CharField(max_length=100, verbose_name="record commit", help_text="record commit") add_time = models.DateTimeField(default=timezone.now, verbose_name="test added time") @@ -157,6 +162,24 @@ class Meta: verbose_name_plural = "test dataset" +class PGTransaction(models.Model): + test_dataset = models.ForeignKey(TestDataSet, verbose_name="test dataset id", help_text="test dataset id") + scale = models.IntegerField(verbose_name="scale", help_text="current scale factor") + num_branches = models.IntegerField(verbose_name="number of branches", help_text="number of branches") + num_tellers = models.IntegerField(verbose_name="number of tellers", help_text="number of tellers") + num_accounts = models.IntegerField(verbose_name="number of accounts", help_text="number of accounts") + aid = models.DecimalField(max_digits=18, decimal_places=8, verbose_name="aid", help_text="aid") + bid = models.DecimalField(max_digits=18, decimal_places=8, verbose_name="bid", help_text="bid") + tid = models.DecimalField(max_digits=18, decimal_places=8, verbose_name="tid", help_text="tid") + delta = models.DecimalField(max_digits=18, decimal_places=8, verbose_name="delta", help_text="delta") + + add_time = models.DateTimeField(default=timezone.now, verbose_name="test dataset time") + + class Meta: + verbose_name = "PG Transaction" + verbose_name_plural = "PG Transaction" + + from django.db.models.signals import pre_save from django.dispatch import receiver @@ -169,7 +192,7 @@ def calc_status(sender, instance, **kwargs): machine_id = instance.test_record.test_machine_id add_time = instance.test_record.add_time branch = instance.test_record.branch - prevRecord = TestRecord.objects.order_by('-add_time').filter(test_machine_id=machine_id,branch=branch, + prevRecord = TestRecord.objects.order_by('-add_time').filter(test_machine_id=machine_id, branch=branch, add_time__lt=add_time).first() if (prevRecord == None): print("prev record not found") @@ -199,7 +222,6 @@ def calc_status(sender, instance, **kwargs): class TestResult(models.Model): - test_dataset = models.ForeignKey(TestDataSet, verbose_name="test dataset id", help_text="test dataset id") latency = models.IntegerField(verbose_name="latency", help_text="latency of the test result") scale = models.IntegerField(verbose_name="scale", help_text="scale of the test result") @@ -224,3 +246,13 @@ class TestResult(models.Model): class Meta: verbose_name = "test result" verbose_name_plural = "test result" + +# transaction type: TPC-B (sort of) ----?? +# scaling factor: 10 ---scale +# query mode: simple ---mode +# number of clients: 10 ---client +# number of threads: 1 ----thread +# number of transactions per client: 1000 -----run??? +# number of transactions actually processed: 10000/10000 +# tps = 85.184871 (including connections establishing) +# tps = 85.296346 (excluding connections establishing) diff --git a/rest_api/apps/records/views.py b/rest_api/apps/records/views.py index a50fe7d..9422ea2 100644 --- a/rest_api/apps/records/views.py +++ b/rest_api/apps/records/views.py @@ -10,6 +10,7 @@ from records.exception import TestDataUploadError from records.filters import TestRecordListFilter from machines.models import Machine +from django.contrib.auth.models import User from records.models import TestCategory, TestBranch from rest_api.settings import DB_ENUM from records.serializers import * @@ -115,7 +116,16 @@ def TestRecordCreate(request, format=None): try: secret = request.META.get("HTTP_AUTHORIZATION") - ret = Machine.objects.filter(machine_secret=secret, state='A').get() + # ret = Machine.objects.filter(machine_secret=secret, state='A').get() + try: + ret = Machine.objects.filter(machine_secret=secret, state='A').get() + except Exception as e: + Machine.objects.create(alias='test', machine_secret=secret, state='A', + owner_id=User.objects.get(username='gsoccamp'), owner_email='test', + owner_username='gsoccamp').save() + ret = Machine.objects.filter(machine_secret=secret, state='A').get() + # print(ret) + test_machine = ret.id if test_machine <= 0: raise TestDataUploadError("The machine is unavailable.") @@ -158,7 +168,12 @@ def TestRecordCreate(request, format=None): if (branch_str == 'master'): branch_str = 'HEAD' - branch = TestBranch.objects.filter(branch_name__iexact=branch_str, is_accept=True).get() + try: + branch = TestBranch.objects.filter(branch_name__iexact=branch_str, is_accept=True).get() + except Exception as e: + TestBranch.objects.create(branch_name=branch_str, is_accept=True).save() + branch = TestBranch.objects.filter(branch_name__iexact=branch_str, is_accept=True).get() + # print(branch) if not branch: raise TestDataUploadError('The branch name is unavailable.') @@ -218,7 +233,12 @@ def TestRecordCreate(request, format=None): #print(tag) #print(tag_list) - test_cate = TestCategory.objects.get(cate_sn=tag) + # test_cate = TestCategory.objects.get(cate_sn=tag) + try: + test_cate = TestCategory.objects.get(cate_sn=tag) + except Exception as e: + TestCategory.objects.create(cate_name='cate1', cate_sn=tag, cate_order=1).save() + test_cate = TestCategory.objects.get(cate_sn=tag) if not test_cate: continue diff --git a/rest_api/apps/users/__pycache__/__init__.cpython-36.pyc b/rest_api/apps/users/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..499a08d Binary files /dev/null and b/rest_api/apps/users/__pycache__/__init__.cpython-36.pyc differ diff --git a/rest_api/apps/users/__pycache__/admin.cpython-36.pyc b/rest_api/apps/users/__pycache__/admin.cpython-36.pyc new file mode 100644 index 0000000..eaef221 Binary files /dev/null and b/rest_api/apps/users/__pycache__/admin.cpython-36.pyc differ diff --git a/rest_api/apps/users/__pycache__/auth.cpython-36.pyc b/rest_api/apps/users/__pycache__/auth.cpython-36.pyc new file mode 100644 index 0000000..81dfebd Binary files /dev/null and b/rest_api/apps/users/__pycache__/auth.cpython-36.pyc differ diff --git a/rest_api/apps/users/__pycache__/filters.cpython-36.pyc b/rest_api/apps/users/__pycache__/filters.cpython-36.pyc new file mode 100644 index 0000000..bf0094c Binary files /dev/null and b/rest_api/apps/users/__pycache__/filters.cpython-36.pyc differ diff --git a/rest_api/apps/users/__pycache__/models.cpython-36.pyc b/rest_api/apps/users/__pycache__/models.cpython-36.pyc new file mode 100644 index 0000000..b17b0f3 Binary files /dev/null and b/rest_api/apps/users/__pycache__/models.cpython-36.pyc differ diff --git a/rest_api/apps/users/__pycache__/serializers.cpython-36.pyc b/rest_api/apps/users/__pycache__/serializers.cpython-36.pyc new file mode 100644 index 0000000..0497da6 Binary files /dev/null and b/rest_api/apps/users/__pycache__/serializers.cpython-36.pyc differ diff --git a/rest_api/apps/users/__pycache__/urls.cpython-36.pyc b/rest_api/apps/users/__pycache__/urls.cpython-36.pyc new file mode 100644 index 0000000..79764c1 Binary files /dev/null and b/rest_api/apps/users/__pycache__/urls.cpython-36.pyc differ diff --git a/rest_api/apps/users/__pycache__/views.cpython-36.pyc b/rest_api/apps/users/__pycache__/views.cpython-36.pyc new file mode 100644 index 0000000..ea107e3 Binary files /dev/null and b/rest_api/apps/users/__pycache__/views.cpython-36.pyc differ diff --git a/rest_api/requirements.txt b/rest_api/requirements.txt index d17208a..a88d49e 100644 --- a/rest_api/requirements.txt +++ b/rest_api/requirements.txt @@ -45,3 +45,4 @@ xlrd==1.1.0 xlwt==1.3.0 psycopg2==2.8.2 pygments==2.4.2 +numpy==1.18.1 \ No newline at end of file diff --git a/rest_api/rest_api/__pycache__/__init__.cpython-36.pyc b/rest_api/rest_api/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..7704eed Binary files /dev/null and b/rest_api/rest_api/__pycache__/__init__.cpython-36.pyc differ diff --git a/rest_api/rest_api/__pycache__/settings.cpython-36.pyc b/rest_api/rest_api/__pycache__/settings.cpython-36.pyc new file mode 100644 index 0000000..5534ce7 Binary files /dev/null and b/rest_api/rest_api/__pycache__/settings.cpython-36.pyc differ diff --git a/rest_api/rest_api/__pycache__/settings_local.cpython-36.pyc b/rest_api/rest_api/__pycache__/settings_local.cpython-36.pyc new file mode 100644 index 0000000..5ec680d Binary files /dev/null and b/rest_api/rest_api/__pycache__/settings_local.cpython-36.pyc differ diff --git a/rest_api/rest_api/__pycache__/urls.cpython-36.pyc b/rest_api/rest_api/__pycache__/urls.cpython-36.pyc new file mode 100644 index 0000000..f597ba1 Binary files /dev/null and b/rest_api/rest_api/__pycache__/urls.cpython-36.pyc differ diff --git a/rest_api/rest_api/__pycache__/wsgi.cpython-36.pyc b/rest_api/rest_api/__pycache__/wsgi.cpython-36.pyc new file mode 100644 index 0000000..1541126 Binary files /dev/null and b/rest_api/rest_api/__pycache__/wsgi.cpython-36.pyc differ diff --git a/rest_api/rest_api/settings_local.py b/rest_api/rest_api/settings_local.py index 29bc0bd..d612550 100644 --- a/rest_api/rest_api/settings_local.py +++ b/rest_api/rest_api/settings_local.py @@ -3,11 +3,11 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'postgres', # data_base name - 'USER': 'chenzhang', + 'NAME': 'pgperffarm-db', + 'USER': 'postgres', 'PASSWORD': 'password', - #'HOST': '/var/run/postgresql' - 'HOST': '/tmp' + 'HOST': '/var/run/postgresql' + #'HOST': '/raid' } } @@ -19,4 +19,4 @@ PGAUTH_KEY = '' EMAIL_HOST_USER = '' -EMAIL_HOST_PASSWORD = '' # individual smtp password +EMAIL_HOST_PASSWORD = '' # individual smtp password \ No newline at end of file diff --git a/rest_api/rest_api/settings_local.py.in b/rest_api/rest_api/settings_local.py.in index de91134..41d131e 100644 --- a/rest_api/rest_api/settings_local.py.in +++ b/rest_api/rest_api/settings_local.py.in @@ -7,7 +7,7 @@ DATABASES = { 'USER': 'chenzhang', 'PASSWORD': 'password', #'HOST': '/var/run/postgresql' - 'HOST': '/tmp' + 'HOST': '/raid' } }