Add support for syncing all file types for both pcwatch and pcsync

README.md

@@ -4,8 +4,9 @@ The `pcsync` and `pcwatch` utilities allow editing copies of
 JavaScript and other textual files of a PlayCanvas project
 locally on your own computer, in a text editor of your choice.
 
-`pcsync` also allows pushing and pulling of [binary files](#using-pcsync-for-binary-files), such as
-images and models.
+Both `pcsync` and `pcwatch` support working with [binary files](#using-pcsync-for-binary-files), such as
+images and models. `pcsync` can push and pull binary files, while `pcwatch` can monitor and
+automatically sync changes to binary files in real-time.
 
 In addition, if your project has a file called [`pcignore.txt`](#the-pcignoretxt-file),
 PlayCanvas merge will not affect the files listed in it,
@@ -70,7 +71,26 @@ automatically answer "yes" to confirmation prompts.
 
 # The `pcwatch` Utility
 
-`pcwatch` does not need any options.
+`pcwatch` detects changes to local files and folders (edits, removals and creation)
+as they happen and applies them to PlayCanvas in real time.
+
+By default, `pcwatch` only monitors textual files. To monitor binary files, use:
+
+```
+  -e, --ext <extensions>  handle files with provided extensions
+  -r, --regexp <regexp>   handle files matching the provided regular expression
+  -a, --all               handle all files (textual and binary)
+```
+
+For instance:
+
+```
+pcwatch -e jpeg,png
+pcwatch -r "\\.(png|jpeg)"
+pcwatch --all
+```
+
+The regular expression tests each file's path from the root.
 
 Moving or renaming a file or a folder
 will appear to `pcwatch` as a `remove + create`. In such cases it may be better to
@@ -144,23 +164,69 @@ Binary files include assets such as textures (JPG and PNG) and models (GLB).
 
 `push`, `pull` (single file) and `rm` work with binary file arguments without any special options.
 
-`pushAll`, `pullAll` and `diffAll` have two options that make them work with matching
+`pushAll`, `pullAll` and `diffAll` have three options that make them work with matching
 files only, including binary (without one of these options `pcsync` only works with textual files):
 
 ```
   -e, --ext <extensions>  handle files with provided extensions
   -r, --regexp <regexp>   handle files matching the provided regular expression
+  -a, --all               handle all files (textual and binary)
 ```
 
 For instance:
 
 ```
 pcsync diffAll -e jpeg,png
 pcsync pushAll -r "\\.(png|jpeg)"
+pcsync pullAll --all
 ```
 
 The regular expression tests each file's path from the root.
 
+## Pulling All Files
+
+To pull all files (both textual and binary) without specifying file types:
+
+```bash
+# Pull all files - easiest option
+pcsync pullAll --all
+
+# Alternative: Use regex to match all files
+pcsync pullAll -r ".*"
+
+# Skip confirmation prompt
+pcsync pullAll --all -y
+```
+
+The `--all` flag is the simplest way to work with all file types without having to manually specify extensions or regex patterns.
+
+# Using `pcwatch` for Binary Files
+
+Similar to `pcsync`, `pcwatch` can monitor and automatically sync binary files such as textures (JPG and PNG) and models (GLB) in real-time.
+
+To use `pcwatch` with binary files, use the same options as `pcsync`:
+
+```
+  -e, --ext <extensions>  handle files with provided extensions
+  -r, --regexp <regexp>   handle files matching the provided regular expression
+  -a, --all               handle all files (textual and binary)
+```
+
+For instance:
+
+```
+pcwatch -e jpeg,png,glb
+pcwatch -r "\\.(png|jpeg|glb)$"
+pcwatch --all
+```
+
+The easiest way to monitor all file types is to use the `--all` option.
+
+When monitoring binary files, `pcwatch` will:
+- Detect when binary files are created, modified, or deleted
+- Automatically upload changes to PlayCanvas in real-time
+- Use the same MD5 hash comparison as `pcsync` to detect actual file changes
+
 # Installation
 
 Use a recent stable version of `node`. We recommend using `nvm`.
@@ -299,7 +365,7 @@ to print the current values of all config variables and other useful data.
 
 * Run `pcsync pullAll` to download existing textual files
   from PlayCanvas
-* Launch `pcwatch`
+* Launch `pcwatch` (or `pcwatch --all` to also monitor binary files)
 * Start editing/creating files locally in your own text editor
 
 To merge changes from another PlayCanvas branch into your branch without `git`:
@@ -316,7 +382,7 @@ To merge changes from another PlayCanvas branch into your branch without `git`:
 * Create a git branch for your work, and make it your local target directory
 * Create a [`pcignore.txt`](#the-pcignoretxt-file) file, listing all files 
 you intend to keep in git, create a PlayCanvas checkpoint that includes your `pcignore.txt`
-* Launch `pcwatch`
+* Launch `pcwatch` (or `pcwatch --all` to also monitor binary files)
 * Start editing/creating files locally in your own text editor
 * When necessary, merge in `git` the branch of another group member into your branch 
 * Use `pcsync pushAll` to update your remote branch after git merge

bin/pcsync.js

@@ -13,6 +13,7 @@ program
     .description('compare all local and remote files and folders')
     .option('-r, --regexp <regexp>', 'handle files matching the provided regular expression')
     .option('-e, --ext <extensions>', 'handle files with provided extensions')
+    .option('-a, --all', 'handle all files (textual and binary)')
     .action(runCompAll);
 
 program
@@ -25,6 +26,7 @@ program
     .description('download all remote files, overwriting their local counterparts')
     .option('-r, --regexp <regexp>', 'handle files matching the provided regular expression')
     .option('-e, --ext <extensions>', 'handle files with provided extensions')
+    .option('-a, --all', 'handle all files (textual and binary)')
     .option('-y, --yes', 'Automatically answer "yes" to any prompts that might print on the command line.')
     .action(runOverwriteAllLocal);
 
@@ -33,6 +35,7 @@ program
     .description('upload all local files, overwriting their remote counterparts')
     .option('-r, --regexp <regexp>', 'handle files matching the provided regular expression')
     .option('-e, --ext <extensions>', 'handle files with provided extensions')
+    .option('-a, --all', 'handle all files (textual and binary)')
     .option('-y, --yes', 'Automatically answer "yes" to any prompts that might print on the command line.')
     .action(runOverwriteAllRemote);
 

bin/pcwatch.js

@@ -12,10 +12,16 @@ const CacheUtils = require('../src/utils/cache-utils.js');
 const LocalTraversal = require('../src/utils/local-traversal.js');
 
 program.option('-f, --force', 'skip local/remote equality check');
+program.option('-r, --regexp <regexp>', 'handle files matching the provided regular expression');
+program.option('-e, --ext <extensions>', 'handle files with provided extensions');
+program.option('-a, --all', 'handle all files (textual and binary)');
 
 program.parse(process.argv);
 
 async function run() {
+    // Handle binary file options
+    CUtils.handleForceRegOpts(program.opts());
+
     if (!program.opts().force) {
         await CUtils.wrapUserErrors(() => SyncUtils.errorIfDifferent(true));
 
@@ -63,28 +69,35 @@ async function handleGoodEvent(e, conf) {
         await eventModified(e, conf);
 
     } else if (e.action === 'ACTION_DELETED') {
-        await WatchUtils.actionDeleted(e.remotePath, conf);
-
-        console.log(`Deleted ${e.remotePath}`);
+        await eventDeleted(e, conf);
 
     } else if (e.action === 'ACTION_CREATED') {
         await eventCreated(e, conf);
     }
 }
 
+async function eventDeleted(e, conf) {
+    const deleted = await WatchUtils.actionDeleted(e.remotePath, conf);
+    if (deleted) {
+        WatchUtils.reportWatchAction(e.remotePath, 'Deleted', conf);
+    } else {
+        WatchUtils.reportWatchAction(e.remotePath, 'Deleted locally. Doesn\'t exist on remote.', conf);
+    }
+}
+
 async function eventModified(e, conf) {
     if (CUtils.eventHasAsset(e, conf)) {
-        const id = await WatchUtils.actionModified(e, conf);
+        await WatchUtils.actionModified(e, conf);
 
-        WatchUtils.reportWatchAction(id, 'Updated', conf);
+        WatchUtils.reportWatchAction(e.remotePath, 'Updated', conf);
     }
 }
 
 async function eventCreated(e, conf) {
     if (!CUtils.eventHasAsset(e, conf)) {
-        const id = await new ActionCreated(e, conf).run();
+        await new ActionCreated(e, conf).run();
 
-        WatchUtils.reportWatchAction(id, 'Created', conf);
+        WatchUtils.reportWatchAction(e.remotePath, 'Created', conf);
     }
 }
 

package.json

@@ -33,14 +33,13 @@
     "src/diff/diff_match_patch_uncompressed.js"
   ],
   "dependencies": {
+    "axios": "^1.7.7",
     "bottleneck": "^2.19.5",
     "commander": "^9.0.0",
     "diff": "^5.0.0",
     "find-process": "^1.4.7",
     "gitignore-parser": "0.0.2",
-    "mkdirp": "^1.0.4",
-    "request": "^2.88.2",
-    "request-promise-native": "^1.0.9"
+    "mkdirp": "^1.0.4"
   },
   "devDependencies": {
     "@playcanvas/eslint-config": "^1.7.1",

src/sync-commands/compute-diff-all.js

@@ -1,3 +1,5 @@
+const path = require('path');
+const fs = require('fs');
 const GetConfig = require('../utils/get-config.js');
 const CUtils = require('../utils/common-utils.js');
 const CacheUtils = require('../utils/cache-utils.js');
@@ -103,10 +105,72 @@ class ComputeDiffAll {
 
     selectRemoteOnly(field) {
         this.res.extraItems.remote[field] = this.remote[field].filter((h) => {
-            return !this.commonPaths[field][h.remotePath];
+            // First check if the file exists locally (original logic)
+            if (this.commonPaths[field][h.remotePath]) {
+                return false;
+            }
+
+            // For files, check if they can actually be downloaded
+            if (field === 'files') {
+                return this.canDownloadFile(h);
+            }
+
+            return true;
         });
     }
 
+    canDownloadFile(h) {
+        // Skip assets that don't have downloadable files (folders, containers, etc.)
+        if (h.type === 'folder' || !h.file) {
+            return false;
+        }
+
+        // Check if downloading this file would cause a path conflict
+        if (this.wouldCausePathConflict(h)) {
+            return false;
+        }
+
+        return true;
+    }
+
+    wouldCausePathConflict(h) {
+        // Get the full local path where this file would be saved
+        const fullPath = CUtils.assetToFullPath(h, this.conf);
+        const dir = path.dirname(fullPath);
+
+        // Check if the immediate parent directory is actually a file
+        if (fs.existsSync(dir)) {
+            const stat = fs.statSync(dir);
+            if (stat.isFile()) {
+                return true;
+            }
+        }
+
+        // Check if any parent directory in the path is actually a file
+        const pathParts = dir.split(path.sep);
+        let currentPath = '';
+
+        for (let i = 0; i < pathParts.length; i++) {
+            if (pathParts[i] === '') {
+                currentPath = path.sep; // Handle root path
+                continue;
+            }
+
+            currentPath = currentPath === path.sep ?
+                path.join(currentPath, pathParts[i]) :
+                path.join(currentPath, pathParts[i]);
+
+            if (fs.existsSync(currentPath)) {
+                const stat = fs.statSync(currentPath);
+                if (stat.isFile()) {
+                    return true;
+                }
+            }
+        }
+
+        return false;
+    }
+
     addToCommonPaths(a, field) {
         a.forEach((h) => {
             this.commonPaths[field][h.remotePath] = 1;

src/sync-commands/overwrite-all-local-with-remote.js

@@ -48,6 +48,11 @@ class OverwriteAllLocalWithRemote {
     async fetchFile(h, action) {
         const asset = this.conf.store.pathToAsset[h.remotePath];
 
+        // Skip assets that don't have downloadable files (folders, containers, etc.)
+        if (asset.type === 'folder' || !asset.file) {
+            return;
+        }
+
         await this.conf.client.loadAssetToFile(asset, this.conf);
 
         this.actionEnd(action, h);

src/utils/common-utils.js

@@ -62,6 +62,44 @@ const CUtils = {
     },
 
     streamToFile: function (readStream, file) {
+        // Check for path conflicts before creating the file
+        const dir = path.dirname(file);
+
+        // Check if the immediate parent directory is actually a file
+        if (fs.existsSync(dir)) {
+            const stat = fs.statSync(dir);
+            if (stat.isFile()) {
+                console.warn(`Skipping ${file} - parent path ${dir} is a file, not a directory`);
+                return Promise.resolve();
+            }
+        }
+
+        // Check if any parent directory in the path is actually a file
+        const pathParts = dir.split(path.sep);
+        let currentPath = '';
+
+        for (let i = 0; i < pathParts.length; i++) {
+            if (pathParts[i] === '') {
+                currentPath = path.sep; // Handle root path
+                continue;
+            }
+
+            currentPath = currentPath === path.sep ?
+                path.join(currentPath, pathParts[i]) :
+                path.join(currentPath, pathParts[i]);
+
+            if (fs.existsSync(currentPath)) {
+                const stat = fs.statSync(currentPath);
+                if (stat.isFile()) {
+                    console.warn(`Skipping ${file} - parent path ${currentPath} is a file, not a directory`);
+                    return Promise.resolve();
+                }
+            }
+        }
+
+        // Create directory structure if it doesn't exist
+        CUtils.makeDirP(dir);
+
         const writeStream = fs.createWriteStream(file);
 
         readStream.pipe(writeStream);
@@ -173,7 +211,7 @@ const CUtils = {
     sameHashAsRemote: async function (local, conf) {
         const remote = conf.store.pathToAsset[local.remotePath];
 
-        if (remote) {
+        if (remote && remote.file) {
             const md5 = await CUtils.fileToMd5Hash(local.fullPath);
 
             return md5 === remote.file.hash;
@@ -247,9 +285,14 @@ const CUtils = {
     },
 
     handleForceRegOpts: function (cmdObj) {
-        const v = cmdObj.regexp ||
+        let v = cmdObj.regexp ||
             (cmdObj.ext && CUtils.extToReg(cmdObj.ext));
 
+        // Handle --all option to match all files
+        if (cmdObj.all && !v) {
+            v = '.*'; // Match all files
+        }
+
         CUtils.checkSetEnv('PLAYCANVAS_FORCE_REG', v);
     },