diff --git a/.idea/.gitignore b/.idea/.gitignore
new file mode 100644
index 0000000..e7e9d11
--- /dev/null
+++ b/.idea/.gitignore
@@ -0,0 +1,2 @@
+# Default ignored files
+/workspace.xml
diff --git a/.idea/django-postgres-stack.iml b/.idea/django-postgres-stack.iml
new file mode 100644
index 0000000..ff86ada
--- /dev/null
+++ b/.idea/django-postgres-stack.iml
@@ -0,0 +1,13 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
new file mode 100644
index 0000000..105ce2d
--- /dev/null
+++ b/.idea/inspectionProfiles/profiles_settings.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 0000000..739fced
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,7 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 0000000..82ae627
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 0000000..94a25f7
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/client/__pycache__/settings.cpython-36.pyc b/client/__pycache__/settings.cpython-36.pyc
new file mode 100644
index 0000000..dc7e9d9
Binary files /dev/null and b/client/__pycache__/settings.cpython-36.pyc differ
diff --git a/client/__pycache__/settings_local.cpython-36.pyc b/client/__pycache__/settings_local.cpython-36.pyc
new file mode 100644
index 0000000..4017133
Binary files /dev/null and b/client/__pycache__/settings_local.cpython-36.pyc differ
diff --git a/client/benchmarks/__pycache__/__init__.cpython-36.pyc b/client/benchmarks/__pycache__/__init__.cpython-36.pyc
new file mode 100644
index 0000000..f6970ef
Binary files /dev/null and b/client/benchmarks/__pycache__/__init__.cpython-36.pyc differ
diff --git a/client/benchmarks/__pycache__/pgbench.cpython-36.pyc b/client/benchmarks/__pycache__/pgbench.cpython-36.pyc
new file mode 100644
index 0000000..8ace00b
Binary files /dev/null and b/client/benchmarks/__pycache__/pgbench.cpython-36.pyc differ
diff --git a/client/benchmarks/__pycache__/runner.cpython-36.pyc b/client/benchmarks/__pycache__/runner.cpython-36.pyc
new file mode 100644
index 0000000..1559e76
Binary files /dev/null and b/client/benchmarks/__pycache__/runner.cpython-36.pyc differ
diff --git a/client/benchmarks/pgbench.py b/client/benchmarks/pgbench.py
index 3cf2719..3e0d775 100644
--- a/client/benchmarks/pgbench.py
+++ b/client/benchmarks/pgbench.py
@@ -214,7 +214,7 @@ def run_tests(self, csv_queue):
# derive configuration for the CPU count / RAM size
configs = PgBench._configure(cpu_count(), available_ram())
- results = {'ro': {}, 'rw': {}}
+ results = {'ro': {}, 'rw': {}} #ro:read only rw:read-write
j = 0
for config in configs:
scale = config['scale']
@@ -223,6 +223,7 @@ def run_tests(self, csv_queue):
results['ro'][scale] = {}
if scale not in results['rw']:
results['rw'][scale] = {}
+ #print(results)
# init for the dataset scale and warmup
self._init(scale)
diff --git a/client/benchmarks/runner.py b/client/benchmarks/runner.py
index 8544fac..b4580fe 100644
--- a/client/benchmarks/runner.py
+++ b/client/benchmarks/runner.py
@@ -83,7 +83,7 @@ def _run_config(self, config_name):
# expand the attribute names
bench = bench(**config['config'])
- self._cluster.start(config=config['postgres'])
+ self._cluster.start(config=config['postgres']) #obtain the database name pgperffarm-db l44 'postgres': postgres_config,
# start collector(s) of additional info
self._collector.start()
@@ -143,8 +143,10 @@ def _upload_results(self, results):
post.append(postdata)
headers = {'Content-Type': 'application/json; charset=utf-8', 'Authorization': self._secret}
+ print(1)
r = requests.post(self._url.encode('utf-8'), data=json.dumps(post).encode('utf-8'), headers=headers)
-
+ print(self._url)
+ # print(post)
def run(self):
'run all the configured benchmarks'
diff --git a/client/collectors/__pycache__/__init__.cpython-36.pyc b/client/collectors/__pycache__/__init__.cpython-36.pyc
new file mode 100644
index 0000000..4667920
Binary files /dev/null and b/client/collectors/__pycache__/__init__.cpython-36.pyc differ
diff --git a/client/collectors/__pycache__/collectd.cpython-36.pyc b/client/collectors/__pycache__/collectd.cpython-36.pyc
new file mode 100644
index 0000000..8edc1bd
Binary files /dev/null and b/client/collectors/__pycache__/collectd.cpython-36.pyc differ
diff --git a/client/collectors/__pycache__/collector.cpython-36.pyc b/client/collectors/__pycache__/collector.cpython-36.pyc
new file mode 100644
index 0000000..1253f56
Binary files /dev/null and b/client/collectors/__pycache__/collector.cpython-36.pyc differ
diff --git a/client/collectors/__pycache__/linux.cpython-36.pyc b/client/collectors/__pycache__/linux.cpython-36.pyc
new file mode 100644
index 0000000..f096a39
Binary files /dev/null and b/client/collectors/__pycache__/linux.cpython-36.pyc differ
diff --git a/client/collectors/__pycache__/postgres.cpython-36.pyc b/client/collectors/__pycache__/postgres.cpython-36.pyc
new file mode 100644
index 0000000..58df105
Binary files /dev/null and b/client/collectors/__pycache__/postgres.cpython-36.pyc differ
diff --git a/client/collectors/collectd.py b/client/collectors/collectd.py
index 1cbe3b8..143983e 100644
--- a/client/collectors/collectd.py
+++ b/client/collectors/collectd.py
@@ -3,8 +3,8 @@
from utils.logging import log
from utils.misc import run_cmd
-COLLECTD_CONFIG = '/tmp/.collectd.conf'
-COLLECTD_PIDFILE = '/tmp/.collectd.pid'
+COLLECTD_CONFIG = '/raid/.collectd.conf'
+COLLECTD_PIDFILE = '/raid/.collectd.pid'
class CollectdCollector(object):
diff --git a/client/perffarm-client.py b/client/perffarm-client.py
index a0e4c7d..affe52a 100644
--- a/client/perffarm-client.py
+++ b/client/perffarm-client.py
@@ -19,8 +19,10 @@
from settings_local import *
from settings import *
-API_URL = 'http://127.0.0.1:8000/'
-MACHINE_SECRET = '610f79063e62e6ad09460ac2c4e66da0386dc89b'
+# API_URL = 'http://127.0.0.1:8000/'
+API_URL = 'http://127.0.0.1:8000/upload/'
+#MACHINE_SECRET = '610f79063e62e6ad09460ac2c4e66da0386dc89b'
+MACHINE_SECRET = 'e984c3017cd1a0dff0ef9f0c394a5c285e421411'
if __name__ == '__main__':
with FileLock('.lock') as lock:
@@ -34,8 +36,9 @@
'''
# clone repository and build the sources
+ print(0000000000000000000000000)
repository = GitRepository(url=GIT_URL, path=REPOSITORY_PATH)
- print(repository.current_branch())
+ print(repository.current_branch())#we are in django-...,but not postgre repo
#if GIT_CLONE:
# repository.clone_or_update()
@@ -70,8 +73,8 @@
PGBENCH_CONFIG['results_dir'] = OUTPUT_DIR
runner.register_config('pgbench-basic',
'pgbench',
- repository.current_branch(),
- repository.current_commit(),
+ 'master',
+ 'akfhdfwiowhqgjog',
dbname=DATABASE_NAME,
bin_path=('%s/bin' % (BUILD_PATH,)),
postgres_config=POSTGRES_CONFIG,
diff --git a/client/settings.py b/client/settings.py
index 0a4f31f..cafad66 100644
--- a/client/settings.py
+++ b/client/settings.py
@@ -2,11 +2,13 @@
import sys
# global configuration
-GIT_URL = 'https://github.com/postgres/postgres.git'
-REPOSITORY_PATH = '/tmp/git-postgres'
-BUILD_PATH = '/Users/chenzhang/anaconda3'
+GIT_URL = 'https://gitee.com/purpleyu/postgres.git'
+REPOSITORY_PATH = '/raid/git-postgres'
+# REPOSITORY_PATH = '/home/guo/Documents/git-postgres/postgres'
+BUILD_PATH = '/usr/lib/postgresql/11'
BIN_PATH = os.path.join(BUILD_PATH, 'bin')
-DATADIR_PATH = '/tmp/data-postgres'
+DATADIR_PATH = '/raid/data-postgres'
+# DATADIR_PATH = '/home/guo/Documents/git-postgres/postgres'
POSTGRES_CONFIG = {
'shared_buffers': '1GB',
@@ -22,9 +24,9 @@
'checkpoint_completion_target': '0.9',
}
-DATABASE_NAME = 'postgres' # This name needs to be the same as rest_api settings_local.py database NAME
+DATABASE_NAME = 'pgperffarm-db' # This name needs to be the same as rest_api settings_local.py database NAME
-OUTPUT_DIR = '/tmp/perf-output'
+OUTPUT_DIR = '/raid/perf-output'
# configuration for PgBench
# runs - number of repetitions (including test for all client counts)
diff --git a/client/utils/__pycache__/__init__.cpython-36.pyc b/client/utils/__pycache__/__init__.cpython-36.pyc
new file mode 100644
index 0000000..f6cb89d
Binary files /dev/null and b/client/utils/__pycache__/__init__.cpython-36.pyc differ
diff --git a/client/utils/__pycache__/cluster.cpython-36.pyc b/client/utils/__pycache__/cluster.cpython-36.pyc
new file mode 100644
index 0000000..1eb18e5
Binary files /dev/null and b/client/utils/__pycache__/cluster.cpython-36.pyc differ
diff --git a/client/utils/__pycache__/git.cpython-36.pyc b/client/utils/__pycache__/git.cpython-36.pyc
new file mode 100644
index 0000000..92e85c8
Binary files /dev/null and b/client/utils/__pycache__/git.cpython-36.pyc differ
diff --git a/client/utils/__pycache__/locking.cpython-36.pyc b/client/utils/__pycache__/locking.cpython-36.pyc
new file mode 100644
index 0000000..00660c4
Binary files /dev/null and b/client/utils/__pycache__/locking.cpython-36.pyc differ
diff --git a/client/utils/__pycache__/logging.cpython-36.pyc b/client/utils/__pycache__/logging.cpython-36.pyc
new file mode 100644
index 0000000..ffc6c98
Binary files /dev/null and b/client/utils/__pycache__/logging.cpython-36.pyc differ
diff --git a/client/utils/__pycache__/misc.cpython-36.pyc b/client/utils/__pycache__/misc.cpython-36.pyc
new file mode 100644
index 0000000..65b997d
Binary files /dev/null and b/client/utils/__pycache__/misc.cpython-36.pyc differ
diff --git a/rest_api/apps/machines/__pycache__/__init__.cpython-36.pyc b/rest_api/apps/machines/__pycache__/__init__.cpython-36.pyc
new file mode 100644
index 0000000..3ced7f3
Binary files /dev/null and b/rest_api/apps/machines/__pycache__/__init__.cpython-36.pyc differ
diff --git a/rest_api/apps/machines/__pycache__/admin.cpython-36.pyc b/rest_api/apps/machines/__pycache__/admin.cpython-36.pyc
new file mode 100644
index 0000000..6f56006
Binary files /dev/null and b/rest_api/apps/machines/__pycache__/admin.cpython-36.pyc differ
diff --git a/rest_api/apps/machines/__pycache__/models.cpython-36.pyc b/rest_api/apps/machines/__pycache__/models.cpython-36.pyc
new file mode 100644
index 0000000..c38faf9
Binary files /dev/null and b/rest_api/apps/machines/__pycache__/models.cpython-36.pyc differ
diff --git a/rest_api/apps/machines/__pycache__/permissions.cpython-36.pyc b/rest_api/apps/machines/__pycache__/permissions.cpython-36.pyc
new file mode 100644
index 0000000..2924fcb
Binary files /dev/null and b/rest_api/apps/machines/__pycache__/permissions.cpython-36.pyc differ
diff --git a/rest_api/apps/machines/__pycache__/serializers.cpython-36.pyc b/rest_api/apps/machines/__pycache__/serializers.cpython-36.pyc
new file mode 100644
index 0000000..2e6dbf9
Binary files /dev/null and b/rest_api/apps/machines/__pycache__/serializers.cpython-36.pyc differ
diff --git a/rest_api/apps/machines/__pycache__/urls.cpython-36.pyc b/rest_api/apps/machines/__pycache__/urls.cpython-36.pyc
new file mode 100644
index 0000000..5231aa9
Binary files /dev/null and b/rest_api/apps/machines/__pycache__/urls.cpython-36.pyc differ
diff --git a/rest_api/apps/machines/__pycache__/views.cpython-36.pyc b/rest_api/apps/machines/__pycache__/views.cpython-36.pyc
new file mode 100644
index 0000000..601b80f
Binary files /dev/null and b/rest_api/apps/machines/__pycache__/views.cpython-36.pyc differ
diff --git a/rest_api/apps/machines/migrations/0001_initial.py b/rest_api/apps/machines/migrations/0001_initial.py
new file mode 100644
index 0000000..d17ad8c
--- /dev/null
+++ b/rest_api/apps/machines/migrations/0001_initial.py
@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.17 on 2020-02-03 17:22
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='Alias',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=32, unique=True, verbose_name='alias name')),
+ ('is_used', models.BooleanField(default=False, verbose_name='is_used')),
+ ('add_time', models.DateTimeField(auto_now_add=True)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='Machine',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('add_time', models.DateTimeField(auto_now_add=True)),
+ ('alias', models.CharField(blank=True, default='', max_length=100)),
+ ('machine_secret', models.CharField(blank=True, default='', max_length=32, verbose_name='machine secret')),
+ ('sn', models.CharField(blank=True, default='', max_length=16)),
+ ('os_name', models.CharField(blank=True, default='', max_length=100)),
+ ('os_version', models.CharField(blank=True, default='', max_length=100)),
+ ('comp_name', models.CharField(blank=True, default='', max_length=100)),
+ ('comp_version', models.CharField(blank=True, default='', max_length=100)),
+ ('state', models.CharField(choices=[('A', 'Active'), ('I', 'Inactive')], default='A', max_length=10)),
+ ('owner_email', models.EmailField(max_length=256, verbose_name='email')),
+ ('owner_username', models.CharField(blank=True, default='', max_length=100)),
+ ('owner_id', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='machines', to=settings.AUTH_USER_MODEL)),
+ ],
+ options={
+ 'ordering': ('add_time',),
+ },
+ ),
+ ]
diff --git a/rest_api/apps/machines/migrations/0002_auto_20200213_1523.py b/rest_api/apps/machines/migrations/0002_auto_20200213_1523.py
new file mode 100644
index 0000000..813015f
--- /dev/null
+++ b/rest_api/apps/machines/migrations/0002_auto_20200213_1523.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.17 on 2020-02-13 15:23
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('machines', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='machine',
+ name='machine_secret',
+ field=models.CharField(blank=True, default='', max_length=100, verbose_name='machine secret'),
+ ),
+ ]
diff --git a/rest_api/apps/machines/migrations/__init__.py b/rest_api/apps/machines/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/rest_api/apps/machines/migrations/__pycache__/0001_initial.cpython-36.pyc b/rest_api/apps/machines/migrations/__pycache__/0001_initial.cpython-36.pyc
new file mode 100644
index 0000000..617499c
Binary files /dev/null and b/rest_api/apps/machines/migrations/__pycache__/0001_initial.cpython-36.pyc differ
diff --git a/rest_api/apps/machines/migrations/__pycache__/0002_auto_20200213_1523.cpython-36.pyc b/rest_api/apps/machines/migrations/__pycache__/0002_auto_20200213_1523.cpython-36.pyc
new file mode 100644
index 0000000..4dbb598
Binary files /dev/null and b/rest_api/apps/machines/migrations/__pycache__/0002_auto_20200213_1523.cpython-36.pyc differ
diff --git a/rest_api/apps/machines/migrations/__pycache__/__init__.cpython-36.pyc b/rest_api/apps/machines/migrations/__pycache__/__init__.cpython-36.pyc
new file mode 100644
index 0000000..0851e01
Binary files /dev/null and b/rest_api/apps/machines/migrations/__pycache__/__init__.cpython-36.pyc differ
diff --git a/rest_api/apps/machines/models.py b/rest_api/apps/machines/models.py
index c9bc731..d45f10f 100644
--- a/rest_api/apps/machines/models.py
+++ b/rest_api/apps/machines/models.py
@@ -29,7 +29,7 @@ class Machine(models.Model):
add_time = models.DateTimeField(auto_now_add=True)
alias = models.CharField(max_length=100, blank=True, default='')
- machine_secret = models.CharField(max_length=32, blank=True, default='', verbose_name="machine secret")
+ machine_secret = models.CharField(max_length=100, blank=True, default='', verbose_name="machine secret")
sn = models.CharField(max_length=16, blank=True, default='')
os_name = models.CharField(max_length=100, blank=True, default='')
os_version = models.CharField(max_length=100, blank=True, default='')
diff --git a/rest_api/apps/records/__pycache__/__init__.cpython-36.pyc b/rest_api/apps/records/__pycache__/__init__.cpython-36.pyc
new file mode 100644
index 0000000..29cb07a
Binary files /dev/null and b/rest_api/apps/records/__pycache__/__init__.cpython-36.pyc differ
diff --git a/rest_api/apps/records/__pycache__/admin.cpython-36.pyc b/rest_api/apps/records/__pycache__/admin.cpython-36.pyc
new file mode 100644
index 0000000..a9137f9
Binary files /dev/null and b/rest_api/apps/records/__pycache__/admin.cpython-36.pyc differ
diff --git a/rest_api/apps/records/__pycache__/exception.cpython-36.pyc b/rest_api/apps/records/__pycache__/exception.cpython-36.pyc
new file mode 100644
index 0000000..6380ffe
Binary files /dev/null and b/rest_api/apps/records/__pycache__/exception.cpython-36.pyc differ
diff --git a/rest_api/apps/records/__pycache__/filters.cpython-36.pyc b/rest_api/apps/records/__pycache__/filters.cpython-36.pyc
new file mode 100644
index 0000000..050a956
Binary files /dev/null and b/rest_api/apps/records/__pycache__/filters.cpython-36.pyc differ
diff --git a/rest_api/apps/records/__pycache__/models.cpython-36.pyc b/rest_api/apps/records/__pycache__/models.cpython-36.pyc
new file mode 100644
index 0000000..629538b
Binary files /dev/null and b/rest_api/apps/records/__pycache__/models.cpython-36.pyc differ
diff --git a/rest_api/apps/records/__pycache__/serializers.cpython-36.pyc b/rest_api/apps/records/__pycache__/serializers.cpython-36.pyc
new file mode 100644
index 0000000..1d14c40
Binary files /dev/null and b/rest_api/apps/records/__pycache__/serializers.cpython-36.pyc differ
diff --git a/rest_api/apps/records/__pycache__/urls.cpython-36.pyc b/rest_api/apps/records/__pycache__/urls.cpython-36.pyc
new file mode 100644
index 0000000..9f674c5
Binary files /dev/null and b/rest_api/apps/records/__pycache__/urls.cpython-36.pyc differ
diff --git a/rest_api/apps/records/__pycache__/views.cpython-36.pyc b/rest_api/apps/records/__pycache__/views.cpython-36.pyc
new file mode 100644
index 0000000..e2e6d7c
Binary files /dev/null and b/rest_api/apps/records/__pycache__/views.cpython-36.pyc differ
diff --git a/rest_api/apps/records/migrations/0001_initial.py b/rest_api/apps/records/migrations/0001_initial.py
new file mode 100644
index 0000000..05db8e8
--- /dev/null
+++ b/rest_api/apps/records/migrations/0001_initial.py
@@ -0,0 +1,164 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.17 on 2020-02-03 17:22
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ('machines', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='LinuxInfo',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('mounts', models.TextField(default='null', help_text='mounts', verbose_name='mounts')),
+ ('cpuinfo', models.TextField(default='null', help_text='cpuinfo', verbose_name='cpuinfo')),
+ ('sysctl', models.TextField(default='null', help_text='sysctl', verbose_name='sysctl')),
+ ('meminfo', models.TextField(default='null', help_text='meminfo', verbose_name='meminfo')),
+ ],
+ options={
+ 'verbose_name': 'linux info',
+ 'verbose_name_plural': 'linux info',
+ },
+ ),
+ migrations.CreateModel(
+ name='MetaInfo',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('date', models.DateTimeField(help_text='date', verbose_name='date')),
+ ('uname', models.TextField(help_text='uname', verbose_name='uname')),
+ ('benchmark', models.TextField(help_text='benchmark', verbose_name='benchmark')),
+ ('name', models.TextField(help_text='name', verbose_name='name')),
+ ],
+ options={
+ 'verbose_name': 'meta info',
+ 'verbose_name_plural': 'meta info',
+ },
+ ),
+ migrations.CreateModel(
+ name='PGInfo',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('checkpoint_timeout', models.CharField(help_text='checkpoint_timeout', max_length=32, verbose_name='checkpoint_timeout')),
+ ('log_temp_files', models.IntegerField(help_text='log_temp_files', verbose_name='log_temp_files')),
+ ('work_mem', models.CharField(help_text='work_mem', max_length=32, verbose_name='work_mem')),
+ ('log_line_prefix', models.CharField(help_text='checkpoint_timeout', max_length=64, verbose_name='checkpoint_timeout')),
+ ('shared_buffers', models.CharField(help_text='shared_buffers', max_length=32, verbose_name='shared_buffers')),
+ ('log_autovacuum_min_duration', models.IntegerField(help_text='log_autovacuum_min_duration', verbose_name='log_autovacuum_min_duration')),
+ ('checkpoint_completion_target', models.DecimalField(decimal_places=4, help_text='checkpoint_completion_target', max_digits=8, verbose_name='checkpoint_completion_target')),
+ ('maintenance_work_mem', models.CharField(help_text='maintenance_work_mem', max_length=32, verbose_name='maintenance_work_mem')),
+ ('log_checkpoints', models.IntegerField(choices=[(1, 'on'), (2, 'off')], help_text='log_checkpoints', verbose_name='log_checkpoints')),
+ ('max_wal_size', models.CharField(help_text='max_wal_size', max_length=32, verbose_name='max_wal_size')),
+ ('min_wal_size', models.CharField(help_text='min_wal_size', max_length=32, verbose_name='min_wal_size')),
+ ],
+ options={
+ 'verbose_name': 'pg info',
+ 'verbose_name_plural': 'pg info',
+ },
+ ),
+ migrations.CreateModel(
+ name='TestBranch',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('branch_name', models.CharField(help_text='branch name', max_length=128, unique=True, verbose_name='branch name')),
+ ('branch_order', models.IntegerField(default=5, help_text='order in all the branch', verbose_name='branch order')),
+ ('is_show', models.BooleanField(default=True, help_text='branch isshow', verbose_name='branch is shown')),
+ ('is_accept', models.BooleanField(default=True, help_text='branch accepts new reports', verbose_name='branch accepts new reports')),
+ ('add_time', models.DateTimeField(default=django.utils.timezone.now, help_text='branch added time', verbose_name='branch added time')),
+ ],
+ options={
+ 'verbose_name': 'test branch',
+ 'verbose_name_plural': 'test branch',
+ },
+ ),
+ migrations.CreateModel(
+ name='TestCategory',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('cate_name', models.CharField(help_text='cate name', max_length=64, verbose_name='cate name')),
+ ('cate_sn', models.CharField(help_text='cate sn', max_length=32, unique=True, verbose_name='cate sn')),
+ ('cate_order', models.IntegerField(help_text='order in the current level', verbose_name='cate order')),
+ ('add_time', models.DateTimeField(default=django.utils.timezone.now, help_text='category added time', verbose_name='add time')),
+ ],
+ options={
+ 'verbose_name': 'tests category',
+ 'verbose_name_plural': 'tests category',
+ },
+ ),
+ migrations.CreateModel(
+ name='TestDataSet',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('clients', models.IntegerField(help_text='clients of the test dataset', verbose_name='clients')),
+ ('scale', models.IntegerField(help_text='scale of the test dataset', verbose_name='scale')),
+ ('std', models.DecimalField(decimal_places=8, help_text='std of the test dataset', max_digits=18, verbose_name='std')),
+ ('metric', models.DecimalField(decimal_places=8, help_text='metric of the test dataset', max_digits=18, verbose_name='metric')),
+ ('median', models.DecimalField(decimal_places=8, help_text='median of the test dataset', max_digits=18, verbose_name='median')),
+ ('status', models.IntegerField(choices=[(-1, 'none'), (1, 'improved'), (2, 'quo'), (3, 'regressive')], help_text='status of this dataset', verbose_name='status')),
+ ('percentage', models.DecimalField(decimal_places=4, help_text='percentage compared to previous dataset', max_digits=8, verbose_name='percentage')),
+ ('add_time', models.DateTimeField(default=django.utils.timezone.now, verbose_name='test dataset time')),
+ ('prev', models.ForeignKey(blank=True, help_text='previous test dataset id', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='prev1', to='records.TestDataSet', verbose_name='previous test dataset id')),
+ ('test_cate', models.ForeignKey(help_text='test cate id', on_delete=django.db.models.deletion.CASCADE, to='records.TestCategory', verbose_name='test cate id')),
+ ],
+ options={
+ 'verbose_name': 'test dataset',
+ 'verbose_name_plural': 'test dataset',
+ },
+ ),
+ migrations.CreateModel(
+ name='TestRecord',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('test_desc', models.TextField(help_text='test desc', verbose_name='test desc')),
+ ('meta_time', models.DateTimeField(default=django.utils.timezone.now, verbose_name='meta time')),
+ ('hash', models.CharField(default='', help_text='record hash', max_length=128, unique=True, verbose_name='record hash')),
+ ('uuid', models.CharField(default='', help_text='record uuid', max_length=64, unique=True, verbose_name='record uuid')),
+ ('commit', models.CharField(help_text='record commit', max_length=64, verbose_name='record commit')),
+ ('add_time', models.DateTimeField(default=django.utils.timezone.now, verbose_name='test added time')),
+ ('branch', models.ForeignKey(help_text='pg branch', on_delete=django.db.models.deletion.CASCADE, to='records.TestBranch', verbose_name='pg branch')),
+ ('linux_info', models.ForeignKey(help_text='linux info', on_delete=django.db.models.deletion.CASCADE, to='records.LinuxInfo', verbose_name='linux info')),
+ ('meta_info', models.ForeignKey(help_text='meta info', on_delete=django.db.models.deletion.CASCADE, to='records.MetaInfo', verbose_name='meta info')),
+ ('pg_info', models.ForeignKey(help_text='pg info', on_delete=django.db.models.deletion.CASCADE, to='records.PGInfo', verbose_name='pg info')),
+ ('test_machine', models.ForeignKey(help_text='person who add this test item', on_delete=django.db.models.deletion.CASCADE, to='machines.Machine', verbose_name='test owner')),
+ ],
+ options={
+ 'verbose_name': 'tests',
+ 'verbose_name_plural': 'tests',
+ },
+ ),
+ migrations.CreateModel(
+ name='TestResult',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('latency', models.IntegerField(help_text='latency of the test result', verbose_name='latency')),
+ ('scale', models.IntegerField(help_text='scale of the test result', verbose_name='scale')),
+ ('end', models.DecimalField(decimal_places=12, help_text='endtime of the test result', max_digits=32, verbose_name='end')),
+ ('clients', models.IntegerField(help_text='clients of the test result', verbose_name='clients')),
+ ('start', models.DecimalField(decimal_places=12, help_text='starttime of the test result', max_digits=32, verbose_name='start')),
+ ('tps', models.DecimalField(decimal_places=6, default=0, help_text='tps of the test result', max_digits=18, verbose_name='tps')),
+ ('run', models.IntegerField(help_text='run number', verbose_name='run')),
+ ('threads', models.IntegerField(help_text='threads of the test result', verbose_name='threads')),
+ ('mode', models.IntegerField(choices=[(1, 'simple'), (2, 'other'), (-1, 'test')], help_text='test mode', verbose_name='mode')),
+ ('add_time', models.DateTimeField(default=django.utils.timezone.now, verbose_name='test result added time')),
+ ('test_dataset', models.ForeignKey(help_text='test dataset id', on_delete=django.db.models.deletion.CASCADE, to='records.TestDataSet', verbose_name='test dataset id')),
+ ],
+ options={
+ 'verbose_name': 'test result',
+ 'verbose_name_plural': 'test result',
+ },
+ ),
+ migrations.AddField(
+ model_name='testdataset',
+ name='test_record',
+ field=models.ForeignKey(help_text='test record id', on_delete=django.db.models.deletion.CASCADE, to='records.TestRecord', verbose_name='test record id'),
+ ),
+ ]
diff --git a/rest_api/apps/records/migrations/0002_auto_20200215_1749.py b/rest_api/apps/records/migrations/0002_auto_20200215_1749.py
new file mode 100644
index 0000000..65a4849
--- /dev/null
+++ b/rest_api/apps/records/migrations/0002_auto_20200215_1749.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.17 on 2020-02-15 17:49
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('records', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='testrecord',
+ name='commit',
+ field=models.CharField(help_text='record commit', max_length=100, verbose_name='record commit'),
+ ),
+ ]
diff --git a/rest_api/apps/records/migrations/__init__.py b/rest_api/apps/records/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/rest_api/apps/records/migrations/__pycache__/0001_initial.cpython-36.pyc b/rest_api/apps/records/migrations/__pycache__/0001_initial.cpython-36.pyc
new file mode 100644
index 0000000..6e49cd7
Binary files /dev/null and b/rest_api/apps/records/migrations/__pycache__/0001_initial.cpython-36.pyc differ
diff --git a/rest_api/apps/records/migrations/__pycache__/0002_auto_20200215_1749.cpython-36.pyc b/rest_api/apps/records/migrations/__pycache__/0002_auto_20200215_1749.cpython-36.pyc
new file mode 100644
index 0000000..785d507
Binary files /dev/null and b/rest_api/apps/records/migrations/__pycache__/0002_auto_20200215_1749.cpython-36.pyc differ
diff --git a/rest_api/apps/records/migrations/__pycache__/__init__.cpython-36.pyc b/rest_api/apps/records/migrations/__pycache__/__init__.cpython-36.pyc
new file mode 100644
index 0000000..f4d7bb7
Binary files /dev/null and b/rest_api/apps/records/migrations/__pycache__/__init__.cpython-36.pyc differ
diff --git a/rest_api/apps/records/models (copy).py b/rest_api/apps/records/models (copy).py
new file mode 100644
index 0000000..cc12103
--- /dev/null
+++ b/rest_api/apps/records/models (copy).py
@@ -0,0 +1,226 @@
+from datetime import datetime
+from django.utils import timezone
+from django.db import models
+
+# Create your models here.
+from machines.models import Machine
+
+
+class TestBranch(models.Model):
+ """
+ test branch
+ """
+ branch_name = models.CharField(max_length=128, unique=True,verbose_name="branch name", help_text="branch name")
+ branch_order = models.IntegerField(default=5,verbose_name="branch order", help_text="order in all the branch")
+ is_show = models.BooleanField(verbose_name="branch is shown", default=True, help_text="branch isshow")
+ is_accept = models.BooleanField(verbose_name="branch accepts new reports", default=True, help_text="branch accepts new reports")
+ add_time = models.DateTimeField(default=timezone.now, verbose_name="branch added time",
+ help_text="branch added time")
+
+ class Meta:
+ verbose_name = "test branch"
+ verbose_name_plural = "test branch"
+
+ def __str__(self):
+ return self.branch_name
+
+
+class TestCategory(models.Model):
+ """
+ tests category
+ """
+ cate_name = models.CharField(max_length=64, verbose_name="cate name", help_text="cate name")
+ cate_sn = models.CharField(max_length=32, unique=True, verbose_name="cate sn", help_text="cate sn")
+ cate_order = models.IntegerField(verbose_name="cate order", help_text="order in the current level")
+ add_time = models.DateTimeField(default=timezone.now, verbose_name="add time", help_text="category added time")
+
+ class Meta:
+ verbose_name = "tests category"
+ verbose_name_plural = "tests category"
+
+ def __str__(self):
+ return self.cate_name
+
+
+class PGInfo(models.Model):
+
+ checkpoint_timeout = models.CharField(max_length=32, verbose_name="checkpoint_timeout", help_text="checkpoint_timeout")
+ log_temp_files = models.IntegerField(verbose_name="log_temp_files", help_text="log_temp_files")
+ work_mem = models.CharField(max_length=32, verbose_name="work_mem", help_text="work_mem")
+ log_line_prefix = models.CharField(max_length=64,verbose_name="checkpoint_timeout", help_text="checkpoint_timeout")
+ shared_buffers = models.CharField(max_length=32, verbose_name="shared_buffers", help_text="shared_buffers")
+ log_autovacuum_min_duration =models.IntegerField(verbose_name="log_autovacuum_min_duration", help_text="log_autovacuum_min_duration")
+
+
+ checkpoint_completion_target = models.DecimalField(max_digits=8, decimal_places=4,verbose_name="checkpoint_completion_target", help_text="checkpoint_completion_target")
+ maintenance_work_mem = models.CharField(max_length=32, verbose_name="maintenance_work_mem", help_text="maintenance_work_mem")
+
+ SWITCH_CHOICE = (
+ (1, 'on'),
+ (2, 'off')
+ )
+ log_checkpoints = models.IntegerField(choices=SWITCH_CHOICE,verbose_name="log_checkpoints", help_text="log_checkpoints")
+ max_wal_size = models.CharField(max_length=32, verbose_name="max_wal_size", help_text="max_wal_size")
+ min_wal_size = models.CharField(max_length=32, verbose_name="min_wal_size", help_text="min_wal_size")
+
+ # pg_branch = models.ForeignKey(TestBranch, verbose_name="pg branch", help_text="pg branch")
+
+ class Meta:
+ verbose_name = "pg info"
+ verbose_name_plural = "pg info"
+
+
+class MetaInfo(models.Model):
+ """
+ meta info
+ """
+ date = models.DateTimeField(verbose_name="date", help_text="date")
+ uname = models.TextField(verbose_name="uname", help_text="uname")
+ benchmark = models.TextField(verbose_name="benchmark", help_text="benchmark")
+ name = models.TextField(verbose_name="name", help_text="name")
+
+ class Meta:
+ verbose_name = "meta info"
+ verbose_name_plural = "meta info"
+
+
+class LinuxInfo(models.Model):
+ """
+ linux info
+ """
+ mounts = models.TextField(verbose_name="mounts", help_text="mounts", default="null")
+ cpuinfo = models.TextField(verbose_name="cpuinfo", help_text="cpuinfo", default="null")
+ sysctl = models.TextField(verbose_name="sysctl", help_text="sysctl", default="null")
+ meminfo = models.TextField(verbose_name="meminfo", help_text="meminfo", default="null")
+
+ class Meta:
+ verbose_name = "linux info"
+ verbose_name_plural = "linux info"
+
+ def __str__(self):
+ return self.mounts
+
+
+class TestRecord(models.Model):
+ """
+ test record
+ """
+ branch = models.ForeignKey(TestBranch, verbose_name="pg branch", help_text="pg branch")
+ test_machine = models.ForeignKey(Machine, verbose_name="test owner",
+ help_text="person who add this test item")
+ pg_info = models.ForeignKey(PGInfo, verbose_name="pg info", help_text="pg info")
+ meta_info = models.ForeignKey(MetaInfo, verbose_name="meta info", help_text="meta info")
+ linux_info = models.ForeignKey(LinuxInfo, verbose_name="linux info", help_text="linux info")
+ test_desc = models.TextField(verbose_name="test desc", help_text="test desc")
+ # test_branch_id = models.ForeignKey(TestBranch, verbose_name="test category", help_text="test category")
+ meta_time = models.DateTimeField(default=timezone.now, verbose_name="meta time")
+ hash = models.CharField(unique=True, default='', max_length=128, verbose_name="record hash",
+ help_text="record hash")
+ uuid = models.CharField(unique=True, default='', max_length=64, verbose_name="record uuid", help_text="record uuid")
+ commit = models.CharField(max_length=100, verbose_name="record commit", help_text="record commit")
+
+ add_time = models.DateTimeField(default=timezone.now, verbose_name="test added time")
+
+ class Meta:
+ verbose_name = "tests"
+ verbose_name_plural = "tests"
+
+
+class TestDataSet(models.Model):
+ test_record = models.ForeignKey(TestRecord, verbose_name="test record id", help_text="test record id")
+ test_cate = models.ForeignKey(TestCategory, verbose_name="test cate id", help_text="test cate id")
+ clients = models.IntegerField(verbose_name="clients", help_text="clients of the test dataset")
+ scale = models.IntegerField(verbose_name="scale", help_text="scale of the test dataset")
+ std = models.DecimalField(max_digits=18, decimal_places=8, verbose_name="std", help_text="std of the test dataset")
+ metric = models.DecimalField(max_digits=18, decimal_places=8, verbose_name="metric",
+ help_text="metric of the test dataset")
+ median = models.DecimalField(max_digits=18, decimal_places=8, verbose_name="median",
+ help_text="median of the test dataset")
+
+ STATUS_CHOICE = (
+ (-1, 'none'),
+ (1, 'improved'),
+ (2, 'quo'),
+ (3, 'regressive'),
+ )
+ status = models.IntegerField(choices=STATUS_CHOICE, verbose_name="status", help_text="status of this dataset")
+ percentage = models.DecimalField(max_digits=8, decimal_places=4, verbose_name="percentage",
+ help_text="percentage compared to previous dataset")
+
+ prev = models.ForeignKey('self', blank=True, null=True, related_name='prev1',
+ verbose_name="previous test dataset id", help_text="previous test dataset id")
+ # prev = models.ForeignKey('self',verbose_name="previous test dataset id", help_text="previous test dataset id")
+ add_time = models.DateTimeField(default=timezone.now, verbose_name="test dataset time")
+
+ class Meta:
+ verbose_name = "test dataset"
+ verbose_name_plural = "test dataset"
+
+
+from django.db.models.signals import pre_save
+from django.dispatch import receiver
+
+
+@receiver(pre_save, sender=TestDataSet)
+def calc_status(sender, instance, **kwargs):
+ print('dataset: ' + str(instance.id))
+ print('previous: ' + str(instance.prev) + ' will be saved')
+
+ machine_id = instance.test_record.test_machine_id
+ add_time = instance.test_record.add_time
+ branch = instance.test_record.branch
+ prevRecord = TestRecord.objects.order_by('-add_time').filter(test_machine_id=machine_id,branch=branch,
+ add_time__lt=add_time).first()
+ if (prevRecord == None):
+ print("prev record not found")
+ return
+
+ prevTestDataSet = TestDataSet.objects.filter(test_record_id=prevRecord.id, scale=instance.scale,
+ clients=instance.clients, test_cate_id=instance.test_cate_id).first()
+
+ if (prevTestDataSet == None):
+ return
+
+ percentage = (instance.metric - prevTestDataSet.metric) / prevTestDataSet.metric
+
+ status = 0
+ if (percentage >= 0.05):
+ status = 1
+ elif (percentage <= -0.05):
+ status = 3
+ else:
+ status = 2
+
+ instance.percentage = percentage
+ instance.status = status
+ instance.prev_id = prevTestDataSet.id
+
+ return
+
+
+class TestResult(models.Model):
+
+ test_dataset = models.ForeignKey(TestDataSet, verbose_name="test dataset id", help_text="test dataset id")
+ latency = models.IntegerField(verbose_name="latency", help_text="latency of the test result")
+ scale = models.IntegerField(verbose_name="scale", help_text="scale of the test result")
+ end = models.DecimalField(max_digits=32, decimal_places=12, verbose_name="end",
+ help_text="endtime of the test result")
+ clients = models.IntegerField(verbose_name="clients", help_text="clients of the test result")
+ start = models.DecimalField(max_digits=32, decimal_places=12, verbose_name="start",
+ help_text="starttime of the test result")
+ tps = models.DecimalField(default=0, max_digits=18, decimal_places=6, verbose_name="tps",
+ help_text="tps of the test result")
+ run = models.IntegerField(verbose_name="run", help_text="run number")
+ threads = models.IntegerField(verbose_name="threads", help_text="threads of the test result")
+
+ MODE_CHOICE = (
+ (1, 'simple'),
+ (2, 'other'),
+ (-1, 'test')
+ )
+ mode = models.IntegerField(choices=MODE_CHOICE, verbose_name="mode", help_text="test mode")
+ add_time = models.DateTimeField(default=timezone.now, verbose_name="test result added time")
+
+ class Meta:
+ verbose_name = "test result"
+ verbose_name_plural = "test result"
diff --git a/rest_api/apps/records/models.py b/rest_api/apps/records/models.py
index 64b2e2f..cc12103 100644
--- a/rest_api/apps/records/models.py
+++ b/rest_api/apps/records/models.py
@@ -117,7 +117,7 @@ class TestRecord(models.Model):
hash = models.CharField(unique=True, default='', max_length=128, verbose_name="record hash",
help_text="record hash")
uuid = models.CharField(unique=True, default='', max_length=64, verbose_name="record uuid", help_text="record uuid")
- commit = models.CharField(max_length=64, verbose_name="record commit", help_text="record commit")
+ commit = models.CharField(max_length=100, verbose_name="record commit", help_text="record commit")
add_time = models.DateTimeField(default=timezone.now, verbose_name="test added time")
diff --git a/rest_api/apps/records/views (copy).py b/rest_api/apps/records/views (copy).py
new file mode 100644
index 0000000..682b565
--- /dev/null
+++ b/rest_api/apps/records/views (copy).py
@@ -0,0 +1,303 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import django_filters
+import shortuuid
+
+from django.contrib.auth.hashers import make_password
+from rest_framework.pagination import PageNumberPagination
+
+from records.exception import TestDataUploadError
+from records.filters import TestRecordListFilter
+from machines.models import Machine
+from django.contrib.auth.models import User
+from records.models import TestCategory, TestBranch
+from rest_api.settings import DB_ENUM
+from records.serializers import *
+
+from rest_framework.decorators import api_view, permission_classes
+from rest_framework.response import Response
+from rest_framework import mixins, status, permissions
+
+from rest_framework import viewsets
+from .models import TestRecord
+import json
+
+
+class StandardResultsSetPagination(PageNumberPagination):
+ page_size = 20
+ page_size_query_param = 'page_size'
+ max_page_size = 100
+
+
+class BigResultsSetPagination(PageNumberPagination):
+ page_size = 1000
+ page_size_query_param = 'page_size'
+
+
+class TestBranchListViewSet(viewsets.ModelViewSet):
+ """
+ List test branches
+ """
+
+ queryset = TestBranch.objects.all().order_by('branch_order','add_time')
+ serializer_class = TestBranchSerializer
+ pagination_class = BigResultsSetPagination
+
+
+class TestCategoryViewSet(viewsets.ModelViewSet):
+ """
+ List test categories
+ """
+
+ queryset = TestCategory.objects.all().order_by('cate_name')
+ serializer_class = TestCategorySerializer
+ pagination_class = StandardResultsSetPagination
+
+
+class TestRecordListViewSet(mixins.ListModelMixin, viewsets.GenericViewSet):
+ """
+ List test records
+ """
+
+ queryset = TestRecord.objects.all().order_by('add_time')
+ serializer_class = TestRecordListSerializer
+ pagination_class = StandardResultsSetPagination
+ filter_backends = (django_filters.rest_framework.DjangoFilterBackend,)
+ lookup_field = 'uuid'
+ permission_classes = (permissions.AllowAny, )
+
+
+class TestRecordDetailViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
+ """
+ Detail test records
+ """
+ lookup_field = 'uuid'
+ queryset = TestRecord.objects.all().order_by('add_time')
+ serializer_class = TestRecordDetailSerializer
+ permission_classes = (permissions.AllowAny, )
+
+
+class MachineHistoryRecordViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
+ """
+ Machine info page
+ """
+ lookup_field = 'sn'
+ queryset = Machine.objects.all().order_by('add_time')
+ serializer_class = MachineHistoryRecordSerializer
+ permission_classes = (permissions.AllowAny, )
+
+
+class TestRecordListByBranchViewSet(mixins.ListModelMixin, viewsets.GenericViewSet):
+ """
+ List test records (/status)
+ """
+ queryset = TestRecord.objects.order_by('test_machine__alias','-add_time').distinct('test_machine__alias').all()
+ serializer_class = TestRecordListSerializer
+ pagination_class = StandardResultsSetPagination
+ filter_backends = (django_filters.rest_framework.DjangoFilterBackend,)
+ permission_classes = (permissions.IsAuthenticatedOrReadOnly, )
+
+
+@api_view(['POST'])
+@permission_classes((permissions.AllowAny, ))
+def TestRecordCreate(request, format=None):
+ """
+ Receive data from client
+ """
+ data = request.data
+
+ json_data = json.dumps(data[0], ensure_ascii=False)
+ json_data = json.loads(json_data)
+ # obj = data[0].pgbench
+ # jsLoads = json.loads(data[0])
+
+ from django.db import transaction
+
+ try:
+ secret = request.META.get("HTTP_AUTHORIZATION")
+ # ret = Machine.objects.filter(machine_secret=secret, state='A').get()
+ try:
+ ret = Machine.objects.filter(machine_secret=secret, state='A').get()
+ except Exception as e:
+ Machine.objects.create(alias='test', machine_secret=secret, state='A',
+ owner_id=User.objects.get(username='gsoccamp'), owner_email='test',
+ owner_username='gsoccamp').save()
+ ret = Machine.objects.filter(machine_secret=secret, state='A').get()
+ # print(ret)
+
+ test_machine = ret.id
+ if test_machine <= 0:
+ raise TestDataUploadError("The machine is unavailable.")
+
+ record_hash = make_password(str(json_data), 'pg_perf_farm')
+ r = TestRecord.objects.filter(hash=record_hash).count()
+ if r != 0:
+ raise TestDataUploadError('The same record already exists, please do not submit it twice.')
+
+ with transaction.atomic():
+
+ if 'linux' not in json_data:
+ print('linuxInfo not found')
+ linuxInfo = LinuxInfoSerializer(data={'mounts': 'none', 'cpuinfo': 'none', 'sysctl': 'none', 'meminfo': 'none'})
+
+ else:
+ linux_data = json_data['linux']
+ linuxInfo = LinuxInfoSerializer(data=linux_data)
+ linuxInfoRet = None
+
+ if linuxInfo.is_valid():
+ linuxInfoRet = linuxInfo.save()
+
+ else:
+ msg = 'linuxInfo invalid'
+ raise TestDataUploadError(msg)
+
+ meta_data = json_data['meta']
+ metaInfo = MetaInfoSerializer(data=meta_data)
+ metaInfoRet = None
+ if metaInfo.is_valid():
+ metaInfoRet = metaInfo.save()
+ else:
+ msg = 'metaInfo invalid'
+ raise TestDataUploadError(msg)
+
+ pg_data = json_data['postgres']
+ branch_str = pg_data['branch']
+
+ if (branch_str == 'master'):
+ branch_str = 'HEAD'
+
+ # print("branch", branch_str)
+ # branch = TestBranch.objects.filter(branch_name__iexact=branch_str, is_accept=True).get()
+ # ret = Machine.objects.filter(machine_secret=secret, state='A').get()
+ try:
+ branch = TestBranch.objects.filter(branch_name__iexact=branch_str, is_accept=True).get()
+ except Exception as e:
+ TestBranch.objects.create(branch_name=branch_str, is_accept=True).save()
+ branch = TestBranch.objects.filter(branch_name__iexact=branch_str, is_accept=True).get()
+ # print(branch)
+
+ if not branch:
+ raise TestDataUploadError('The branch name is unavailable.')
+
+ commit = pg_data['commit']
+ pg_settings = pg_data['settings']
+
+ filtered = ['checkpoint_timeout','work_mem','shared_buffers','maintenance_work_mem','max_wal_size','min_wal_size']
+
+ for item in filtered:
+ if item.isdigit():
+ pg_settings[item] = int(pg_settings[item])
+ # pg_settings[item] = pg_settings[item].encode('utf-8')
+ # pg_settings[item] = filter(str.isdigit, pg_settings[item])
+
+ pg_settings['log_checkpoints'] = DB_ENUM['general_switch'][pg_settings['log_checkpoints']]
+ pgInfo = CreatePGInfoSerializer(data=pg_settings)
+ pgInfoRet = None
+
+ if pgInfo.is_valid():
+ pgInfoRet = pgInfo.save()
+
+ else:
+ msg = pgInfo.errors
+ raise TestDataUploadError(msg)
+
+ test_record_data = {
+ 'pg_info': pgInfoRet.id,
+ 'linux_info': linuxInfoRet.id,
+ 'meta_info': metaInfoRet.id,
+ 'test_machine': test_machine,
+ 'test_desc': 'here is desc',
+ 'meta_time': metaInfoRet.date,
+ 'hash': record_hash,
+ 'commit': commit,
+ 'branch': branch.id,
+ 'uuid': shortuuid.uuid()
+ }
+
+ testRecord = CreateTestRecordSerializer(data=test_record_data)
+ testRecordRet = None
+
+ if testRecord.is_valid():
+ testRecordRet = testRecord.save()
+
+ else:
+ msg = 'testRecord invalid'
+ print(testRecord.errors)
+ raise TestDataUploadError(msg)
+
+ pgbench = json_data['pgbench']
+ # print(type(ro))
+ ro = pgbench['ro']
+
+ #for tag, tag_list in pgbench.iteritems():
+ for tag, tag_list in pgbench.items():
+ #print(tag)
+ #print(tag_list)
+
+ # test_cate = TestCategory.objects.get(cate_sn=tag)
+ try:
+ test_cate = TestCategory.objects.get(cate_sn=tag)
+ except Exception as e:
+ TestCategory.objects.create(cate_name='cate1', cate_sn=tag, cate_order=1).save()
+ test_cate = TestCategory.objects.get(cate_sn=tag)
+
+ if not test_cate:
+ continue
+ else:
+ print(test_cate.cate_name)
+
+ for scale, dataset_list in tag_list.items():
+
+ for client_num, dataset in dataset_list.items():
+
+ test_dataset_data = {
+ 'test_record': testRecordRet.id,
+ 'clients': client_num,
+ 'scale': scale,
+ 'std': dataset['std'],
+ 'metric': dataset['metric'],
+ 'median': dataset['median'],
+ 'test_cate': test_cate.id,
+ # status, percentage will calc by receiver
+ 'status': -1,
+ 'percentage': 0.0,
+ }
+
+ testDateSet = CreateTestDateSetSerializer(data=test_dataset_data)
+ testDateSetRet = None
+
+ if testDateSet.is_valid():
+ testDateSetRet = testDateSet.save()
+ else:
+ print(testDateSet.errors)
+ msg = 'testDateSet invalid'
+ raise TestDataUploadError(msg)
+
+ test_result_list = dataset['results']
+
+ for test_result in test_result_list:
+ test_result_data = test_result
+ test_result_data['test_dataset'] = testDateSetRet.id
+ test_result_data['mode'] = DB_ENUM['mode'][test_result_data['mode']]
+ testResult = CreateTestResultSerializer(data=test_result_data)
+
+ testResultRet = None
+
+ if testResult.is_valid():
+ testResultRet = testResult.save()
+
+ else:
+ print(testResult.errors)
+ msg = testResult.errors
+ raise TestDataUploadError(msg)
+
+
+ except Exception as e:
+ msg = 'Upload error: ' + e.__str__()
+ print(msg)
+ return Response(msg, status=status.HTTP_406_NOT_ACCEPTABLE)
+
+ print('Upload successful!')
+ return Response(status=status.HTTP_201_CREATED)
diff --git a/rest_api/apps/records/views.py b/rest_api/apps/records/views.py
index a50fe7d..c4c6708 100644
--- a/rest_api/apps/records/views.py
+++ b/rest_api/apps/records/views.py
@@ -10,6 +10,7 @@
from records.exception import TestDataUploadError
from records.filters import TestRecordListFilter
from machines.models import Machine
+from django.contrib.auth.models import User
from records.models import TestCategory, TestBranch
from rest_api.settings import DB_ENUM
from records.serializers import *
@@ -105,17 +106,29 @@ def TestRecordCreate(request, format=None):
Receive data from client
"""
data = request.data
+ print(22222222)
json_data = json.dumps(data[0], ensure_ascii=False)
json_data = json.loads(json_data)
# obj = data[0].pgbench
# jsLoads = json.loads(data[0])
+ # print(json_data)
from django.db import transaction
try:
secret = request.META.get("HTTP_AUTHORIZATION")
- ret = Machine.objects.filter(machine_secret=secret, state='A').get()
+ print(33333333)
+ # ret = Machine.objects.filter(machine_secret=secret, state='A').get()
+ try:
+ ret = Machine.objects.filter(machine_secret=secret, state='A').get()
+ except Exception as e:
+ Machine.objects.create(alias='test', machine_secret=secret, state='A',
+ owner_id=User.objects.get(username='gsoccamp'), owner_email='test',
+ owner_username='gsoccamp').save()
+ ret = Machine.objects.filter(machine_secret=secret, state='A').get()
+ # print(ret)
+
test_machine = ret.id
if test_machine <= 0:
raise TestDataUploadError("The machine is unavailable.")
@@ -158,7 +171,15 @@ def TestRecordCreate(request, format=None):
if (branch_str == 'master'):
branch_str = 'HEAD'
- branch = TestBranch.objects.filter(branch_name__iexact=branch_str, is_accept=True).get()
+ # print("branch", branch_str)
+ # branch = TestBranch.objects.filter(branch_name__iexact=branch_str, is_accept=True).get()
+ # ret = Machine.objects.filter(machine_secret=secret, state='A').get()
+ try:
+ branch = TestBranch.objects.filter(branch_name__iexact=branch_str, is_accept=True).get()
+ except Exception as e:
+ TestBranch.objects.create(branch_name=branch_str, is_accept=True).save()
+ branch = TestBranch.objects.filter(branch_name__iexact=branch_str, is_accept=True).get()
+ # print(branch)
if not branch:
raise TestDataUploadError('The branch name is unavailable.')
@@ -218,7 +239,12 @@ def TestRecordCreate(request, format=None):
#print(tag)
#print(tag_list)
- test_cate = TestCategory.objects.get(cate_sn=tag)
+ # test_cate = TestCategory.objects.get(cate_sn=tag)
+ try:
+ test_cate = TestCategory.objects.get(cate_sn=tag)
+ except Exception as e:
+ TestCategory.objects.create(cate_name='cate1', cate_sn=tag, cate_order=1).save()
+ test_cate = TestCategory.objects.get(cate_sn=tag)
if not test_cate:
continue
diff --git a/rest_api/apps/users/__pycache__/__init__.cpython-36.pyc b/rest_api/apps/users/__pycache__/__init__.cpython-36.pyc
new file mode 100644
index 0000000..499a08d
Binary files /dev/null and b/rest_api/apps/users/__pycache__/__init__.cpython-36.pyc differ
diff --git a/rest_api/apps/users/__pycache__/admin.cpython-36.pyc b/rest_api/apps/users/__pycache__/admin.cpython-36.pyc
new file mode 100644
index 0000000..eaef221
Binary files /dev/null and b/rest_api/apps/users/__pycache__/admin.cpython-36.pyc differ
diff --git a/rest_api/apps/users/__pycache__/auth.cpython-36.pyc b/rest_api/apps/users/__pycache__/auth.cpython-36.pyc
new file mode 100644
index 0000000..81dfebd
Binary files /dev/null and b/rest_api/apps/users/__pycache__/auth.cpython-36.pyc differ
diff --git a/rest_api/apps/users/__pycache__/filters.cpython-36.pyc b/rest_api/apps/users/__pycache__/filters.cpython-36.pyc
new file mode 100644
index 0000000..bf0094c
Binary files /dev/null and b/rest_api/apps/users/__pycache__/filters.cpython-36.pyc differ
diff --git a/rest_api/apps/users/__pycache__/models.cpython-36.pyc b/rest_api/apps/users/__pycache__/models.cpython-36.pyc
new file mode 100644
index 0000000..b17b0f3
Binary files /dev/null and b/rest_api/apps/users/__pycache__/models.cpython-36.pyc differ
diff --git a/rest_api/apps/users/__pycache__/serializers.cpython-36.pyc b/rest_api/apps/users/__pycache__/serializers.cpython-36.pyc
new file mode 100644
index 0000000..0497da6
Binary files /dev/null and b/rest_api/apps/users/__pycache__/serializers.cpython-36.pyc differ
diff --git a/rest_api/apps/users/__pycache__/urls.cpython-36.pyc b/rest_api/apps/users/__pycache__/urls.cpython-36.pyc
new file mode 100644
index 0000000..79764c1
Binary files /dev/null and b/rest_api/apps/users/__pycache__/urls.cpython-36.pyc differ
diff --git a/rest_api/apps/users/__pycache__/views.cpython-36.pyc b/rest_api/apps/users/__pycache__/views.cpython-36.pyc
new file mode 100644
index 0000000..ea107e3
Binary files /dev/null and b/rest_api/apps/users/__pycache__/views.cpython-36.pyc differ
diff --git a/rest_api/requirements.txt b/rest_api/requirements.txt
index d17208a..a88d49e 100644
--- a/rest_api/requirements.txt
+++ b/rest_api/requirements.txt
@@ -45,3 +45,4 @@ xlrd==1.1.0
xlwt==1.3.0
psycopg2==2.8.2
pygments==2.4.2
+numpy==1.18.1
\ No newline at end of file
diff --git a/rest_api/rest_api/__pycache__/__init__.cpython-36.pyc b/rest_api/rest_api/__pycache__/__init__.cpython-36.pyc
new file mode 100644
index 0000000..7704eed
Binary files /dev/null and b/rest_api/rest_api/__pycache__/__init__.cpython-36.pyc differ
diff --git a/rest_api/rest_api/__pycache__/settings.cpython-36.pyc b/rest_api/rest_api/__pycache__/settings.cpython-36.pyc
new file mode 100644
index 0000000..5534ce7
Binary files /dev/null and b/rest_api/rest_api/__pycache__/settings.cpython-36.pyc differ
diff --git a/rest_api/rest_api/__pycache__/settings_local.cpython-36.pyc b/rest_api/rest_api/__pycache__/settings_local.cpython-36.pyc
new file mode 100644
index 0000000..5ec680d
Binary files /dev/null and b/rest_api/rest_api/__pycache__/settings_local.cpython-36.pyc differ
diff --git a/rest_api/rest_api/__pycache__/urls.cpython-36.pyc b/rest_api/rest_api/__pycache__/urls.cpython-36.pyc
new file mode 100644
index 0000000..f597ba1
Binary files /dev/null and b/rest_api/rest_api/__pycache__/urls.cpython-36.pyc differ
diff --git a/rest_api/rest_api/__pycache__/wsgi.cpython-36.pyc b/rest_api/rest_api/__pycache__/wsgi.cpython-36.pyc
new file mode 100644
index 0000000..1541126
Binary files /dev/null and b/rest_api/rest_api/__pycache__/wsgi.cpython-36.pyc differ
diff --git a/rest_api/rest_api/settings_local.py b/rest_api/rest_api/settings_local.py
index 29bc0bd..d612550 100644
--- a/rest_api/rest_api/settings_local.py
+++ b/rest_api/rest_api/settings_local.py
@@ -3,11 +3,11 @@
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql_psycopg2',
- 'NAME': 'postgres', # data_base name
- 'USER': 'chenzhang',
+ 'NAME': 'pgperffarm-db',
+ 'USER': 'postgres',
'PASSWORD': 'password',
- #'HOST': '/var/run/postgresql'
- 'HOST': '/tmp'
+ 'HOST': '/var/run/postgresql'
+ #'HOST': '/raid'
}
}
@@ -19,4 +19,4 @@
PGAUTH_KEY = ''
EMAIL_HOST_USER = ''
-EMAIL_HOST_PASSWORD = '' # individual smtp password
+EMAIL_HOST_PASSWORD = '' # individual smtp password
\ No newline at end of file
diff --git a/rest_api/rest_api/settings_local.py.in b/rest_api/rest_api/settings_local.py.in
index de91134..41d131e 100644
--- a/rest_api/rest_api/settings_local.py.in
+++ b/rest_api/rest_api/settings_local.py.in
@@ -7,7 +7,7 @@ DATABASES = {
'USER': 'chenzhang',
'PASSWORD': 'password',
#'HOST': '/var/run/postgresql'
- 'HOST': '/tmp'
+ 'HOST': '/raid'
}
}