RSCBC-211: Community version testing

Author: chvck

.github/workflows/tests.yml

@@ -14,19 +14,17 @@ jobs:
 
     strategy:
       matrix:
-        server:
-          - 8.0.0-3716
-          - 7.6.7
-          - 7.2.7
-          - 7.1.6
-          - 7.0.4
-
+        server: [ 8.0.0, 7.6.7, 7.2.7, 7.1.6, 7.0.4, community-8.0.0 ]
+        include:
+          - server: community-8.0.0
+            rcbserverVersion: 8.0.0-community
+    
     runs-on: ubuntu-latest
     steps:
       - name: Install cbdinocluster
         run: |
           mkdir -p "$HOME/bin"
-          wget -nv -O $HOME/bin/cbdinocluster https://github.com/couchbaselabs/cbdinocluster/releases/download/v0.0.41/cbdinocluster-linux-amd64
+          wget -nv -O $HOME/bin/cbdinocluster https://github.com/couchbaselabs/cbdinocluster/releases/download/v0.0.89/cbdinocluster-linux-amd64
           chmod +x $HOME/bin/cbdinocluster
           echo "$HOME/bin" >> $GITHUB_PATH
 
@@ -50,12 +48,20 @@ jobs:
           CBDC_ID=$(cbdinocluster -v alloc --def="${CLUSTERCONFIG}")
           cbdinocluster -v buckets add ${CBDC_ID} default --ram-quota-mb=100 --flush-enabled=true --num-replicas=2
           cbdinocluster -v collections add ${CBDC_ID} default _default test
-          CBDC_CONNSTR=$(cbdinocluster -v connstr --tls $CBDC_ID)
           CBDC_IP=$(cbdinocluster -v ip $CBDC_ID)
           echo "CBDC_ID=$CBDC_ID" >> "$GITHUB_ENV"
-          echo "CBDC_CONNSTR=$CBDC_CONNSTR" >> "$GITHUB_ENV"
           echo "CBDC_IP=$CBDC_IP" >> "$GITHUB_ENV"
 
+      - name: Set connection string
+        run: |
+          if [[ ${{ matrix.server }} == "community*" ]]; then
+            CBDC_CONNSTR=$(cbdinocluster -v connstr --tls $CBDC_ID)
+          else 
+            CBDC_CONNSTR=$(cbdinocluster -v connstr $CBDC_ID)
+          fi
+          
+          echo "CBDC_CONNSTR=$CBDC_CONNSTR" >> "$GITHUB_ENV"
+
       - name: Create fts index
         run: |
           curl -XPUT -H "Content-Type: application/json" -u Administrator:password http://${{ env.CBDC_IP }}:8094/api/index/basic_search_index -d '{"type": "fulltext-index","name": "basic_search_index","sourceType": "gocbcore","sourceName": "default","planParams": {"maxPartitionsPerPIndex": 1024,"indexPartitions": 1},"params": {"doc_config": {"docid_prefix_delim": "","docid_regexp": "","mode": "type_field","type_field": "type"},"mapping": {"analysis": {},"default_analyzer": "standard","default_datetime_parser": "dateTimeOptional","default_field": "_all","default_mapping": {"dynamic": true,"enabled": true},"default_type": "_default","docvalues_dynamic": true,"index_dynamic": true,"store_dynamic": true,"type_field": "_type"},"store": {"indexType": "scorch","segmentVersion": 15}},"sourceParams": {}}' 
@@ -72,8 +78,13 @@ jobs:
           CBDC_CONNSTR: # from above
           RCBDINOID: ${{ env.CBDC_ID }}
           RCBCONNSTR: ${{ env.CBDC_CONNSTR }}
-          RCBSERVER_VERSION: ${{ matrix.server }}
         run: |
+          if [[ -z "${{ matrix.rcbserverVersion }}" ]]; then {
+            echo "RCBSERVER_VERSION=${{ matrix.server }}"
+          else
+            echo "RCBSERVER_VERSION=${{ matrix.rcbserverVersion }}"
+          fi
+          
           cargo test --color=always --no-fail-fast -- --nocapture
 
       - name: Run allocation tests
@@ -83,8 +94,13 @@ jobs:
           CBDC_CONNSTR: # from above
           RCBDINOID: ${{ env.CBDC_ID }}
           RCBCONNSTR: ${{ env.CBDC_CONNSTR }}
-          RCBSERVER_VERSION: ${{ matrix.server }}
         run: |
+          if [[ -z "${{ matrix.rcbserverVersion }}" ]]; then {
+            echo "RCBSERVER_VERSION=${{ matrix.server }}"
+          else
+            echo "RCBSERVER_VERSION=${{ matrix.rcbserverVersion }}"
+          fi
+          
           cargo test --features dhat-heap --test allocations --color=always --no-fail-fast -- --nocapture
 
 #      - name: Collect couchbase logs

sdk/couchbase-core/src/agent_ops.rs

@@ -274,16 +274,6 @@ impl Agent {
         &self,
         opts: &CreateCollectionOptions<'_>,
     ) -> Result<CreateCollectionResponse> {
-        if opts.history_enabled.is_some() {
-            return self.run_with_bucket_feature_check(
-                BucketFeature::NonDedupedHistory,
-                || async {
-                    self.inner.mgmt.create_collection(opts).await
-                },
-                "History retention is not supported - note that the Magma storage engine must be used",
-            ).await;
-        }
-
         self.inner.mgmt.create_collection(opts).await
     }
 
@@ -298,16 +288,6 @@ impl Agent {
         &self,
         opts: &UpdateCollectionOptions<'_>,
     ) -> Result<UpdateCollectionResponse> {
-        if opts.history_enabled.is_some() {
-            return self.run_with_bucket_feature_check(
-                BucketFeature::NonDedupedHistory,
-                || async {
-                    self.inner.mgmt.update_collection(opts).await
-                },
-                "History retention is not supported - note that the Magma storage engine must be used",
-            )
-                .await;
-        }
         self.inner.mgmt.update_collection(opts).await
     }
 

sdk/couchbase-core/src/memdx/datatype.rs

@@ -15,22 +15,19 @@
  *  * limitations under the License.
  *
  */
+use std::convert::From;
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 #[non_exhaustive]
+#[derive(Default)]
 pub enum DataTypeFlag {
+    #[default]
     None,
     Json,
     Compressed,
     Xattrs,
 }
 
-impl Default for DataTypeFlag {
-    fn default() -> Self {
-        Self::None
-    }
-}
-
 impl From<DataTypeFlag> for u8 {
     fn from(value: DataTypeFlag) -> Self {
         match value {

sdk/couchbase-core/tests/common/features.rs

@@ -16,7 +16,7 @@
  *
  */
 
-use crate::common::node_version::NodeVersion;
+use crate::common::node_version::{NodeEdition, NodeVersion};
 use crate::common::test_agent::TestAgent;
 
 const SERVER_VERSION_720: NodeVersion = NodeVersion {
@@ -55,9 +55,10 @@ pub enum TestFeatureCode {
     SearchManagement,
     SearchManagementCollections,
     BucketManagement,
-    CollectionNoExpiry,
+    CollectionMaxExpiry,
     CollectionUpdates,
     HistoryRetention,
+    UserGroups,
 }
 
 impl TestAgent {
@@ -72,12 +73,21 @@ impl TestAgent {
             TestFeatureCode::SearchManagementCollections => {
                 !self.cluster_version.lower(&SERVER_VERSION_762)
             }
-            TestFeatureCode::CollectionNoExpiry => !self.cluster_version.lower(&SERVER_VERSION_762),
+            TestFeatureCode::CollectionMaxExpiry => {
+                self.cluster_version.edition != Some(NodeEdition::Community)
+                    && !self.cluster_version.lower(&SERVER_VERSION_762)
+            }
             TestFeatureCode::CollectionUpdates => {
                 !self.cluster_version.lower(&SERVER_VERSION_722)
                     && !self.cluster_version.equal(&SERVER_VERSION_722)
             }
-            TestFeatureCode::HistoryRetention => !self.cluster_version.lower(&SERVER_VERSION_720),
+            TestFeatureCode::HistoryRetention => {
+                self.cluster_version.edition != Some(NodeEdition::Community)
+                    && !self.cluster_version.lower(&SERVER_VERSION_720)
+            }
+            TestFeatureCode::UserGroups => {
+                self.cluster_version.edition != Some(NodeEdition::Community)
+            }
         }
     }
 }

sdk/couchbase-core/tests/diagnostics.rs

@@ -37,8 +37,8 @@ fn test_ping() {
     run_test(async |mut agent| {
         let opts = PingOptions::new()
             .kv_timeout(Duration::from_millis(1000))
-            .query_timeout(Duration::from_millis(1000))
-            .search_timeout(Duration::from_millis(1000));
+            .query_timeout(Duration::from_millis(75000))
+            .search_timeout(Duration::from_millis(75000));
 
         let report = agent.ping(&opts).await.unwrap();
 

sdk/couchbase-core/tests/mgmt.rs

@@ -16,16 +16,16 @@
  *
  */
 
+use crate::common::features::TestFeatureCode;
 use crate::common::helpers::{
-    create_scope, delete_collection, delete_scope, feature_supported, generate_string_value,
+    create_scope, delete_collection, delete_scope, ensure_manifest, generate_string_value,
     try_until,
 };
 use crate::common::test_config::run_test;
 use common::helpers;
 use couchbase_core::agent::Agent;
 use couchbase_core::cbconfig::CollectionManifest;
-use couchbase_core::features::BucketFeature;
-use couchbase_core::mgmtx::bucket_settings::{BucketSettings, BucketType};
+use couchbase_core::mgmtx::bucket_settings::{BucketSettings, BucketType, StorageBackend};
 use couchbase_core::options::management::{
     CreateBucketOptions, CreateCollectionOptions, DeleteBucketOptions, EnsureBucketOptions,
     GetBucketOptions, GetCollectionManifestOptions, UpdateBucketOptions,
@@ -70,14 +70,32 @@ fn test_scopes() {
     });
 }
 
+#[serial]
 #[test]
-fn test_collections() {
+fn test_collections_history_retention() {
     run_test(async |mut agent| {
-        let history_supported = feature_supported(&agent, BucketFeature::NonDedupedHistory).await;
+        if !agent.supports_feature(&TestFeatureCode::HistoryRetention) {
+            return;
+        }
 
         let scope_name = generate_string_value(10);
         let collection_name = generate_string_value(10);
-        let bucket_name = agent.test_setup_config.bucket.clone();
+        let bucket_name = generate_string_value(10);
+
+        let settings = BucketSettings::default()
+            .bucket_type(BucketType::COUCHBASE)
+            .storage_backend(StorageBackend::MAGMA)
+            .history_retention_seconds(5)
+            .ram_quota_mb(1024);
+
+        let opts = &CreateBucketOptions::new(&bucket_name, &settings);
+
+        agent.create_bucket(opts).await.unwrap();
+
+        agent
+            .ensure_bucket(&EnsureBucketOptions::new(&bucket_name, false))
+            .await
+            .unwrap();
 
         let resp = create_scope(&agent, &bucket_name, &scope_name)
             .await
@@ -86,12 +104,11 @@ fn test_collections() {
 
         helpers::ensure_manifest(&agent, &bucket_name, resp.manifest_uid).await;
 
-        let mut opts =
-            CreateCollectionOptions::new(&bucket_name, &scope_name, &collection_name).max_ttl(25);
-
-        if history_supported {
-            opts = opts.history_enabled(true)
-        };
+        let mut opts = CreateCollectionOptions::new(&bucket_name, &scope_name, &collection_name)
+            .history_enabled(true);
+        if agent.supports_feature(&TestFeatureCode::CollectionMaxExpiry) {
+            opts = opts.max_ttl(25);
+        }
 
         let resp = agent.create_collection(&opts).await.unwrap();
         assert!(!resp.manifest_uid.is_empty());
@@ -106,13 +123,80 @@ fn test_collections() {
 
         assert!(collection_found.is_some());
         let collection_found = collection_found.unwrap();
-        assert_eq!(collection_found.max_ttl, Some(25));
-        if history_supported {
-            assert_eq!(collection_found.history, Some(true));
-        } else {
-            // Depending on server version the collection may have inherited the bucket default.
-            assert!(collection_found.history.is_none() || collection_found.history == Some(false));
+        if agent.supports_feature(&TestFeatureCode::CollectionMaxExpiry) {
+            assert_eq!(collection_found.max_ttl, Some(25));
         }
+        assert_eq!(collection_found.history, Some(true));
+
+        let resp = delete_collection(&agent, &bucket_name, &scope_name, &collection_name)
+            .await
+            .unwrap();
+        assert!(!resp.manifest_uid.is_empty());
+
+        ensure_manifest(&agent, &bucket_name, resp.manifest_uid).await;
+
+        let manifest = get_manifest(&agent, &bucket_name).await.unwrap();
+
+        let mut scope_found = find_scope(&manifest, &scope_name);
+        let scope_found = scope_found.unwrap();
+        let mut collection_found = find_collection(&scope_found, &collection_name);
+        assert!(collection_found.is_none());
+
+        let resp = delete_scope(&agent, &bucket_name, &scope_name)
+            .await
+            .unwrap();
+        assert!(!resp.manifest_uid.is_empty());
+
+        let _ = agent
+            .delete_bucket(&DeleteBucketOptions::new(&bucket_name))
+            .await;
+
+        // We can't fire and forget the delete, the server will error if we try to update a bucket
+        // whilst one is being deleted which could fail other tests.
+        agent
+            .ensure_bucket(&EnsureBucketOptions::new(&bucket_name, true))
+            .await
+            .unwrap();
+    });
+}
+
+#[test]
+fn test_collections() {
+    run_test(async |mut agent| {
+        let scope_name = generate_string_value(10);
+        let collection_name = generate_string_value(10);
+        let bucket_name = agent.test_setup_config.bucket.clone();
+
+        let resp = create_scope(&agent, &bucket_name, &scope_name)
+            .await
+            .unwrap();
+        assert!(!resp.manifest_uid.is_empty());
+
+        ensure_manifest(&agent, &bucket_name, resp.manifest_uid).await;
+
+        let mut opts = CreateCollectionOptions::new(&bucket_name, &scope_name, &collection_name);
+        if agent.supports_feature(&TestFeatureCode::CollectionMaxExpiry) {
+            opts = opts.max_ttl(25);
+        }
+
+        let resp = agent.create_collection(&opts).await.unwrap();
+        assert!(!resp.manifest_uid.is_empty());
+
+        ensure_manifest(&agent, &bucket_name, resp.manifest_uid).await;
+
+        let manifest = get_manifest(&agent, &bucket_name).await.unwrap();
+        assert!(!manifest.uid.is_empty());
+
+        let scope_found = find_scope(&manifest, &scope_name).unwrap();
+        let mut collection_found = find_collection(&scope_found, &collection_name);
+
+        assert!(collection_found.is_some());
+        let collection_found = collection_found.unwrap();
+        if agent.supports_feature(&TestFeatureCode::CollectionMaxExpiry) {
+            assert_eq!(collection_found.max_ttl, Some(25));
+        }
+        // Depending on server version the collection may have inherited the bucket default.
+        assert!(collection_found.history.is_none() || collection_found.history == Some(false));
 
         let resp = delete_collection(&agent, &bucket_name, &scope_name, &collection_name)
             .await
@@ -241,9 +325,11 @@ fn test_update_bucket() {
             .await
             .unwrap();
 
-        let update_settings = BucketSettings::default()
-            .ram_quota_mb(200)
-            .max_ttl(Duration::from_secs(3600));
+        let mut update_settings = BucketSettings::default().ram_quota_mb(200);
+
+        if agent.supports_feature(&TestFeatureCode::CollectionMaxExpiry) {
+            update_settings = update_settings.max_ttl(Duration::from_secs(3600));
+        }
 
         agent
             .update_bucket(&UpdateBucketOptions::new(&bucket_name, &update_settings))
@@ -276,10 +362,13 @@ fn test_update_bucket() {
             .unwrap();
 
         assert_eq!(bucket.bucket_settings.ram_quota_mb, Some(200));
-        assert_eq!(
-            bucket.bucket_settings.max_ttl,
-            Some(Duration::from_secs(3600))
-        );
+
+        if agent.supports_feature(&TestFeatureCode::CollectionMaxExpiry) {
+            assert_eq!(
+                bucket.bucket_settings.max_ttl,
+                Some(Duration::from_secs(3600))
+            );
+        }
     });
 }