goroutine error on query

Hi,

Hope you are all well !

I cannot insert this query, and I have an error

				REPLACE into limostar (   
					id,remote_id,avatar,owner,name,fullname,description,homepage,url,language,watchers,stars,forks,size,pushed,created,starred,service_id,license   
				) VALUES ('96285','22486533','https://avatars2.githubusercontent.com/u/2347603?v=4','00-Evan','shattered-pixel-dungeon','00-Evan/shattered-pixel-dungeon','Traditional roguelike game with pixel-art graphics and simple interface','unkown','https://github.com/00-Evan/shattered-pixel-dungeon.git','Java','623','623','279','29628','2020-01-13 02:41:56  0100 CET','2014-07-31 23:56:31  0200 CEST','2020-02-18 11:32:08  0100 CET','github','unkown')    
"\x02"   
manticore.apibuf{}   
"\x00"   
panic: runtime error: slice bounds out of range [2:0]   
   
goroutine 1 [running]:   
github.com/x0rzkov/go-sdk/manticore.(*apibuf).getLsbWord(0xc000733440, 0x10002)   
	/Users/x0rzkov/go/pkg/mod/github.com/x0rzkov/go-sdk@v0.0.0-20190514070640-c6a45389b87d/manticore/sphinxql.go:330  0x1af   
github.com/x0rzkov/go-sdk/manticore.(*Sqlresult).parseOK(0xc0007334d8, 0xc000733440)   
	/Users/x0rzkov/go/pkg/mod/github.com/x0rzkov/go-sdk@v0.0.0-20190514070640-c6a45389b87d/manticore/sphinxql.go:270  0x84   
github.com/x0rzkov/go-sdk/manticore.(*Sqlresult).parseChain(0xc0007334d8, 0xc008777e40, 0x18)   
	/Users/x0rzkov/go/pkg/mod/github.com/x0rzkov/go-sdk@v0.0.0-20190514070640-c6a45389b87d/manticore/sphinxql.go:58  0x200   
github.com/x0rzkov/go-sdk/manticore.parseSphinxqlAnswer.func1(0xc008777e40, 0xc008770100, 0xc005fb5b88)   
	/Users/x0rzkov/go/pkg/mod/github.com/x0rzkov/go-sdk@v0.0.0-20190514070640-c6a45389b87d/manticore/sphinxql.go:22  0x8c   
github.com/x0rzkov/go-sdk/manticore.(*Client).netQuery(0xc000280880, 0x8, 0xc008777e20, 0x4feb030, 0x4f30860, 0xc0004781c0, 0x1, 0x1)   
	/Users/x0rzkov/go/pkg/mod/github.com/x0rzkov/go-sdk@v0.0.0-20190514070640-c6a45389b87d/manticore/client.go:225  0x1ca   
github.com/x0rzkov/go-sdk/manticore.(*Client).Sphinxql(0xc000280880, 0xc005fb4f00, 0x26e, 0x1, 0x13, 0xc005fb4f00, 0x26e, 0x0)   
	/Users/x0rzkov/go/pkg/mod/github.com/x0rzkov/go-sdk@v0.0.0-20190514070640-c6a45389b87d/manticore/manticore.go:398  0x70   
github.com/lucmski/limo/cmd.sphinxStars(0x5150500, 0xc0000ac000, 0xc0003ccb40, 0x1, 0x1)   
	/Users/x0rzkov/go/src/github.com/lucmski/limo/cmd/sphinxql.go:161  0xb1a   
github.com/lucmski/limo/cmd.glob..func14(0x5a1b760, 0xc0003ccb40, 0x1, 0x1)   
	/Users/x0rzkov/go/src/github.com/lucmski/limo/cmd/sphinxql.go:48  0xca   
github.com/spf13/cobra.(*Command).execute(0x5a1b760, 0xc0003ccaf0, 0x1, 0x1, 0x5a1b760, 0xc0003ccaf0)   
	/Users/x0rzkov/go/pkg/mod/github.com/spf13/cobra@v0.0.6-0.20190805155617-b80588d523ec/command.go:833  0x2aa   
github.com/spf13/cobra.(*Command).ExecuteC(0x5a1afe0, 0x403d4ba, 0x599d8a0, 0xc000000180)   
	/Users/x0rzkov/go/pkg/mod/github.com/spf13/cobra@v0.0.6-0.20190805155617-b80588d523ec/command.go:917  0x2fb   
github.com/spf13/cobra.(*Command).Execute(...)   
	/Users/x0rzkov/go/pkg/mod/github.com/spf13/cobra@v0.0.6-0.20190805155617-b80588d523ec/command.go:867   
github.com/lucmski/limo/cmd.Execute()   
	/Users/x0rzkov/go/src/github.com/lucmski/limo/cmd/root.go:52  0x31   
main.main()   
	/Users/x0rzkov/go/src/github.com/lucmski/limo/main.go:8  0x20   

Manticore Config

#!/bin/sh   
ip=`hostname -i`   
cat << EOF   
index limostar {   
    type = rt   
    rt_mem_limit = 1024M   
    path = /var/lib/manticore/data/limostar   
   
    rt_attr_bigint = remote_id      
    rt_field = avatar         
    rt_attr_string = owner          
    rt_attr_string = name          
    rt_attr_string = url        
    rt_attr_string = language   
    rt_attr_bigint = watchers   
    rt_attr_bigint = stars     
    rt_attr_bigint = forks          
    rt_attr_bigint = size           
    rt_attr_string = license    
    rt_attr_timestamp = pushed         
    rt_attr_timestamp = created        
    rt_attr_timestamp = starred      
    rt_attr_string = service_id   
    rt_attr_multi = tags   
    rt_attr_multi = topics   
    rt_field = fullname   
    rt_field = description   
    rt_field = service_id   
    rt_field = homepage    
    rt_field = url   
    # stored_fields = fullname,description,service_id   
    min_infix_len = 2   
}   
   
indexer   
{   
    mem_limit           = 64M   
}   
   
index limotopic {   
    type = rt   
    rt_mem_limit = 1024M   
           
    path = /var/lib/manticore/data/limotopic   
    rt_attr_bigint = remote_id   
    rt_attr_string = service_id   
    rt_attr_string = url   
    rt_attr_string = tag   
    rt_field = url   
    rt_field = tag   
    rt_field = service_id   
    # stored_fields = tag,url,service_id   
    min_infix_len = 2   
}   
   
index limo   
{   
  type = distributed   
  local = limostar   
  local = limotopic   
}   
   
index pq {   
    type = percolate   
    path = /var/lib/manticore/data/pq   
    min_infix_len = 4   
}   
   
searchd {   
    listen = 9306:mysql41   
    listen = /var/run/mysqld/mysqld.sock:mysql41   
    listen = $ip:9312   
    listen = 9308:http   
   
    # more info about replication you can find at    
    # https://docs.manticoresearch.com/latest/html/replication.html   
    listen = $ip:9315-9325:replication   
    log = /var/log/manticore/searchd.log   
   
    # you can also send query_log to /dev/stdout to be shown in docker logs   
    query_log = /var/log/manticore/query.log   
    read_timeout = 15   
    max_children = 30   
    pid_file = /var/run/manticore/searchd.pid   
    seamless_rotate = 1   
    preopen_indexes = 1   
    unlink_old = 1   
    # workers = thread_pool   
   
    max_matches         = 2000   
    workers             = threads    
    subtree_docs_cache = 8M   
    subtree_hits_cache = 16M   
   
    binlog_path = /var/lib/manticore/data   
    max_packet_size = 128M   
    mysql_version_string = 5.5.21   
    data_dir = /var/lib/manticore/replication   
}   
EOF   

Any reason why ?

Cheers,
X.

[manticoresearch]

@x0rzkov Hello. Can you please provide the script/code snippet which we can use to reproduce this issue?

Hi,

Hope you are all well and thanks for the reply !

I just made you a repository in order to test my use case.
Ref. https://github.com/x0rzkov/manticore-test

How to run:

Pre-requisites

• docker
• docker-compose

Before

tar xvf manticore-dump.sql.tar.gz

Run

docker-compose up --build

Importer (options)

% importer -h
  -h, --help                    display help
  -f, --input-file string       input-file (default "/opt/manticore/data/manticore-dump.sql")
  -m, --manticore-host string   input-file (default "localhost")
  -p, --manticore-port uint16   input-file (default 9312)
  -j, --parallel-jobs int       parallel-jobs (default 2)

Quick important note:
I do not understand how to add the rt_attr_multi tags in the limostar index.
How do you build the query for real time indexes ?

Cheers,
X

[manticoresearch]

Hi @x0rzkov

I get the following error and backtrace:

manticore-importer | time="2020-02-24T06:36:02Z" level=info msg="query:REPLACE into limotopic (id,remote_id,service_id,url,tag) VALUES ('253b66546bf9e540ac02035f014e62e3', '197773483','github','https://github.com/AbduallahAtta/Social-Note.git','room')"
manticore-importer | panic: runtime error: index out of range [1] with length 1
manticore-importer |
manticore-importer | goroutine 7882 [running]:
manticore-importer | encoding/binary.littleEndian.Uint16(...)
manticore-importer | 	/usr/local/go/src/encoding/binary/binary.go:52
manticore-importer | github.com/manticoresoftware/go-sdk/manticore.(*apibuf).getLsbWord(...)
manticore-importer | 	/go/pkg/mod/github.com/manticoresoftware/go-sdk@v0.0.0-20191205035816-0e8dbffac2c9/manticore/sphinxql.go:325
manticore-importer | github.com/manticoresoftware/go-sdk/manticore.(*Sqlresult).parseOK(0xc0000c7d50, 0xc0000c7cb8)
manticore-importer | 	/go/pkg/mod/github.com/manticoresoftware/go-sdk@v0.0.0-20191205035816-0e8dbffac2c9/manticore/sphinxql.go:267 +0x166
manticore-importer | github.com/manticoresoftware/go-sdk/manticore.(*Sqlresult).parseChain(0xc0000c7d50, 0xc00031b460, 0x18)
manticore-importer | 	/go/pkg/mod/github.com/manticoresoftware/go-sdk@v0.0.0-20191205035816-0e8dbffac2c9/manticore/sphinxql.go:56 +0x27c
manticore-importer | github.com/manticoresoftware/go-sdk/manticore.parseSphinxqlAnswer.func1(0xc00031b460, 0xc000310100, 0xc0002dd080)
manticore-importer | 	/go/pkg/mod/github.com/manticoresoftware/go-sdk@v0.0.0-20191205035816-0e8dbffac2c9/manticore/sphinxql.go:20 +0x8c
manticore-importer | github.com/manticoresoftware/go-sdk/manticore.(*Client).netQuery(0xc0000c0000, 0x8, 0xc00031b440, 0x57c6d0, 0x56dae0, 0xc0004c2000, 0x2, 0x2)
manticore-importer | 	/go/pkg/mod/github.com/manticoresoftware/go-sdk@v0.0.0-20191205035816-0e8dbffac2c9/manticore/client.go:225 +0x1ca
manticore-importer | github.com/manticoresoftware/go-sdk/manticore.(*Client).Sphinxql(0xc0000c0000, 0xc00032b080, 0xb4, 0x2, 0x2, 0xb4, 0x0, 0x0)
manticore-importer | 	/go/pkg/mod/github.com/manticoresoftware/go-sdk@v0.0.0-20191205035816-0e8dbffac2c9/manticore/manticore.go:398 +0x70
manticore-importer | main.main.func1(0xc0000b2140, 0xc0000c0280, 0xc0000c0000, 0x0, 0x0)
manticore-importer | 	/go/src/github.com/x0rzkov/manticore-test/main.go:64 +0x17a
manticore-importer | created by main.main
manticore-importer | 	/go/src/github.com/x0rzkov/manticore-test/main.go:58 +0x607
manticore-importer exited with code 2

Since it's different from the mentioned in this issue's body I want to ask - is the the error which concerns you now?

I do not understand how to add the rt_attr_multi tags in the limostar index.

It's already there:

snikolaev@dev:~/manticore-test$ cat manticore.conf |grep -A 20 "index limostar"
index limostar {
    type = rt
    rt_mem_limit = 1024M
    path = /var/lib/manticore/data/limostar

    rt_attr_bigint = remote_id
    rt_field = avatar
    rt_attr_string = owner
    rt_attr_string = name
    rt_attr_string = url
    rt_attr_string = language
    rt_attr_bigint = watchers
    rt_attr_bigint = stars
    rt_attr_bigint = forks
    rt_attr_bigint = size
    rt_attr_string = license
    rt_attr_timestamp = pushed
    rt_attr_timestamp = created
    rt_attr_timestamp = starred
    rt_attr_string = service_id
    rt_attr_multi = tags

How do you build the query for real time indexes ?

What query do you mean?

Now I have another error,

manticore-importer | time="2020-02-24T07:22:08Z" level=info msg="query:REPLACE into limotopic (id,remote_id,service_id,url,tag) VALUES ('5d10614167fa5e88a26b78d1e16283cd', '43965481','github','https://github.com/0010-IO/gealts.git','elixir')"
manticore-importer | time="2020-02-24T07:22:08Z" level=info msg="query error:  REPLACE into limotopic (id,remote_id,service_id,url,tag) VALUES ('5d10614167fa5e88a26b78d1e16283cd', '43965481','github','https://github.com/0010-IO/gealts.git','elixir')"
manticore-importer | time="2020-02-24T07:22:08Z" level=fatal msg=EOF
manticore-importer exited with code 1

So the idea is to check if manticore's go-sdk can absorb lots of data. That's simply it, and this repo is a good occasion to test your sdk.

For How do you build the query for real time indexes ?
What is the sphinxql query to add multiattr to limostar index ? Give me just an example how to add all the tags per star.

Thanks in advance for your insights per advance.

Ps. It is quite urgent as we have to decide if we have to continue to use manticore or not.

[manticoresearch]

Try workers=thread_pool. In this case I get the error as I showed above - "index out of range [1] with length 1" and it seems it's a bug in the SDK.

What is the sphinxql query to add multiattr to limostar index ?

mysql> alter table limostar add column tags2 multi;
Query OK, 0 rows affected (0.07 sec)

mysql> alter table limostar add column tags3 multi64;
Query OK, 0 rows affected (0.04 sec)

mysql> alter table limostar add column tags4 string;
Query OK, 0 rows affected (0.05 sec)

Give me just an example how to add all the tags per star.

mysql> insert into limostar(tags2,tags3,tags4) values((1,2,3),(1,2,3),'a b c');
Query OK, 1 row affected (0.00 sec)

mysql> select tags4 from limostar where tags4 any ('c', 'd');
+-------+
| tags4 |
+-------+
| a b c |
+-------+
1 row in set (0.00 sec)

Hi,

Thanks a lot for the reply, I am working with @x0rzkov and will try out now the query for tags.

When do you think that the sdk can be fixed ?

Best Regards,
Luc Michalski

[manticoresearch]

@lucmichalski

When do you think that the sdk can be fixed ?

I'm afraid not today. Perhaps not even this week. What I can recommend is you could just use a mysql Go library to connect to the mysql's Manticore port (9306 in your docker compose). The binary protocol which the SDK implements might give very insignificant performance benefit, what it gives is mostly just an abstraction layer for more convenient development, but it seems that in your app (at least what you have shared) it may be not that important.

Also if you're looking for max INSERT/REPLACE throughput and you have lots of incoming docs make sure you do batching as it increases the throughput significantly. Here is what we recently measured on one of our customers's single server with standard linux mysql client + GNU parallel:

With batch=10K+32 parallel workers I could get the insert throughput of a single RT index =
14892 docs per second 
13.71MB (of source data + sql overhead) per second
= 1.12TB per day

There’re 32 cores on the server. They were ~1/3 loaded:
[root@sehs01 ms_vs_es]# dstat 30
You did not select any stats, using -cdngy by default.
----total-cpu-usage---- -dsk/total- -net/total- ---paging-- ---system--
usr sys idl wai hiq siq| read  writ| recv  send|  in   out | int   csw
  2   1  97   0   0   0|1165k 1000k|   0     0 |   0     0 |  12k 3815
 35   2  62   0   0   0|   0  9990k|  20k   24k|   0     0 |  74k 6414
 31   1  68   0   0   0|   0    25M|  22k   62k|   0     0 |  81k 4348
 31   3  66   0   0   0|   0  8491k|  23k   73k|   0     0 |  86k 6229
 27   1  72   0   0   0|   0  7496k|  41k  101k|   0     0 |  43k 4175
 28   2  70   0   0   0|  26k 6686k|  21k   27k|   0     0 |  78k 5874
 25   1  74   0   0   0|   0  6049k|  23k   29k|   0     0 |  36k 3749
 27   2  71   0   0   0|1638B 6129k|  20k   18k|   0     0 |  37k 5690
 25   1  74   0   0   0|   0  5736k|  22k   63k|   0     0 |  30k 3641
 27   2  71   0   0   0|   0    11M|  21k  107k|   0     0 |  40k 5527
 25   1  74   0   0   0| 273B   12M|  24k   65k|   0     0 |  30k 3661
 26   2  72   0   0   0| 546B   10M|  16k   15k|   0     0 |  35k 5478
 24   1  75   0   0   0| 956B   11M|  20k   31k|   0     0 |  23k 3553
 26   3  71   0   0   0| 137B 5713k|  15k   70k|   0     0 |  48k 5755
 24   1  75   0   0   0|  13k   16M|  18k   25k|   0     0 |  54k 4340
 37   3  61   0   0   0|1092B   28M|  16k   37k|   0     0 | 105k 7058
 30   1  69   0   0   0| 273B 8930k|  18k  105k|   0     0 |  64k 4160
 30   2  68   0   0   0| 273B 8018k|  31k   76k|   0     0 |  48k 6180
 28   1  71   0   0   0|3277B 8266k|  27k   26k|   0     0 |  73k 4239
 27   2  71   0   0   0| 273B 7182k|  18k   16k|   0     0 |  38k 5732
 25   1  74   0   0   0|   0  7102k|  18k   80k|   0     0 |  27k 3718
 25   2  73   0   0   0| 273B 6775k|  21k   45k|   0     0 |  37k 5721
 10   1  89   0   0   0| 137B 2957k|  23k   23k|   0     0 |  44k 2817

You can additional enable sharding:

10K batch, 32 workers, 8 shards, NO OPTIMIZE:
[root@sehs01 ms_vs_es]# time parallel -j32 --pipepart -a all.10000.sharded.sql "mysql -P8312 -h0"
real    6m22.163s
user    7m6.135s
sys     0m52.999s
or 23951437 / (6*60+22) = 62700 docs per second or 57,75MB per second or 4,75TB per day

The load was:
[root@sehs01 ms_vs_es]# dstat 30
You did not select any stats, using -cdngy by default.
----total-cpu-usage---- -dsk/total- -net/total- ---paging-- ---system--
usr sys idl wai hiq siq| read  writ| recv  send|  in   out | int   csw
  2   1  97   0   0   0|1119k 1079k|   0     0 |   0     0 |  12k 3837
 89   5   6   0   0   0|3004B   47M|  14k   50k|   0     0 | 372k   12k
 88   3   9   0   0   0|2185B   45M|  11k   12k|   0     0 | 243k   11k
 85   4  11   0   0   0|7646B   46M|  12k   13k|   0     0 | 113k   12k
 84   2  13   0   0   0|1911B   47M|  15k   43k|   0     0 |  96k   10k
 82   4  13   0   0   0| 245k   46M|  11k   17k|   0     0 | 141k   12k
 52   2  46   0   0   0| 193k   32M|  12k   33k|   0     0 | 100k 7965
  5   2  93   0   0   0| 410B   11M|  13k   55k|   0     0 |  28k 4542
  4   1  96   0   0   0| 135k   14M|  15k   36k|   0     0 |6699  2771
  5   2  94   0   0   0|2867B 8923k|  11k   33k|   0     0 |8586  4511
  3   0  96   0   0   0| 241k   13M|  15k   18k|   0     0 |4870  2382
  5   2  93   0   0   0| 410B   13M|  13k   28k|   0     0 |  17k 4487
  3   1  96   0   0   0| 878B   16M|  19k   51k|   0     0 |6771  2875 ^C

Caution: it doesn't include time to OPTIMIZE.

It works fine with the mysql driver, even with 15 parallel jobs !

Please, update us when you have fixed the sdk.

Thanks in advance

[klirichek]

Hello.
I've looked into you code and see following pattern: you've created one client, and then try to use it in parallel from bunch of goroutines. That way it will not work anyway: once client connected, you can use it serial way, sending query-by-query and reading response after each of them. There is only one socket opened, and daemon expects one flow of commands; you can't share one and same among parallel works.

So, you just need to create an instance of client for every separate coro. Then each one will have own connection which is not shared with anothers, and the crashes must go out.

I tried with only one thread, and the bug is also triggered, at list with the go-sdk.

[klirichek]

I've just tried minute ago:

cl := NewClient()

res, err := cl.Sphinxql(`REPLACE into limostar (id,remote_id,avatar,owner,name,fullname,description,homepage,url,language,watchers,stars,forks,size,pushed,created,starred,service_id,license) VALUES ('96285','22486533','https://avatars2.githubusercontent.com/u/2347603?v=4','00-Evan','shattered-pixel-dungeon','00-Evan/shattered-pixel-dungeon','Traditional roguelike game with pixel-art graphics and simple interface','unkown','https://github.com/00-Evan/shattered-pixel-dungeon.git','Java','623','623','279','29628','2020-01-13 02:41:56 +0100 CET','2014-07-31 23:56:31 +0200 CEST','2020-02-18 11:32:08 +0100 CET','github','unkown')`)
	fmt.Println(res, err)

and get [Query OK, 1 rows affected] <nil>.

Full test clause looking like this:

func TestClient_Sphinxql_insert(t *testing.T) {

	cl := NewClient()

	_, err := cl.netQuery(commandPersist, buildBoolRequest(true), nil)

	res, err := cl.Sphinxql(`REPLACE into limostar (id,remote_id,avatar,owner,name,fullname,description,homepage,url,language,watchers,stars,forks,size,pushed,created,starred,service_id,license) VALUES ('96285','22486533','https://avatars2.githubusercontent.com/u/2347603?v=4','00-Evan','shattered-pixel-dungeon','00-Evan/shattered-pixel-dungeon','Traditional roguelike game with pixel-art graphics and simple interface','unkown','https://github.com/00-Evan/shattered-pixel-dungeon.git','Java','623','623','279','29628','2020-01-13 02:41:56 +0100 CET','2014-07-31 23:56:31 +0200 CEST','2020-02-18 11:32:08 +0100 CET','github','unkown')`)
	fmt.Println(res, err)
	res, err = cl.Sphinxql(`replace into testrt values(2,'another subject', 'more content', 15)`)
	fmt.Println(res, err)
	res, err = cl.Sphinxql(`replace into testrt values(5,'again subject', 'one more content', 10)`)
	if err != nil {
		fmt.Println(err.Error())
	} else {
		fmt.Println(res)
	}
}

it works as expected.
What is daemon version you run on? May be something wrong there?

the latest version from the docker container