jpg instead of raw compressed frame

.vscode/settings.json

@@ -0,0 +1,49 @@
+{
+    "files.associations": {
+        "array": "cpp",
+        "atomic": "cpp",
+        "*.tcc": "cpp",
+        "cctype": "cpp",
+        "chrono": "cpp",
+        "clocale": "cpp",
+        "cmath": "cpp",
+        "cstdarg": "cpp",
+        "cstddef": "cpp",
+        "cstdint": "cpp",
+        "cstdio": "cpp",
+        "cstdlib": "cpp",
+        "cstring": "cpp",
+        "ctime": "cpp",
+        "cwchar": "cpp",
+        "cwctype": "cpp",
+        "deque": "cpp",
+        "unordered_map": "cpp",
+        "vector": "cpp",
+        "exception": "cpp",
+        "algorithm": "cpp",
+        "memory": "cpp",
+        "memory_resource": "cpp",
+        "optional": "cpp",
+        "ratio": "cpp",
+        "string": "cpp",
+        "string_view": "cpp",
+        "system_error": "cpp",
+        "tuple": "cpp",
+        "type_traits": "cpp",
+        "utility": "cpp",
+        "fstream": "cpp",
+        "initializer_list": "cpp",
+        "iomanip": "cpp",
+        "iosfwd": "cpp",
+        "iostream": "cpp",
+        "istream": "cpp",
+        "limits": "cpp",
+        "new": "cpp",
+        "ostream": "cpp",
+        "sstream": "cpp",
+        "stdexcept": "cpp",
+        "streambuf": "cpp",
+        "thread": "cpp",
+        "typeinfo": "cpp"
+    }
+}

PIwebVNC.cpp

@@ -57,6 +57,10 @@ void handle_sigint(int sig)
     vncServer.stop_service();
     exit(0);
 }
+void conLoop()
+{
+    wss->connections();
+}
 
 int main(int argc, char *argv[])
 {
@@ -71,7 +75,8 @@ int main(int argc, char *argv[])
     wss = &ws;
     xinputs = &input;
     signal(SIGINT, handle_sigint);
-    std::thread t1 = ws.begin(SERVER_PORT);
+    ws.begin(SERVER_PORT);
+    std::thread t1(conLoop);
     std::thread t2(frameLoop);
     ws.onMessage(onMessageCLBK);
     ws.onConnect(firstFrame);

compile.sh

@@ -6,9 +6,9 @@ if [ -z "$SUDO_USER" ]; then
   exit
 fi
 
-apt install -y libx11-dev libxdamage-dev libxfixes-dev libxtst-dev liblz4-dev g++
+apt install -y libx11-dev libxdamage-dev libxfixes-dev libxtst-dev liblz4-dev g++ libjpeg-dev
 
-g++ PIwebVNC.cpp -lX11 -lXdamage -lXfixes -pthread -lXtst -llz4 -o /bin/PiWebVNC
+g++ PIwebVNC.cpp -lX11 -lXdamage -lXfixes -pthread -lXtst -llz4 -ljpeg -o /bin/PiWebVNC
 
 echo "[INFO] Compile Successful."
 

libs/appConfigs.hpp

@@ -15,7 +15,9 @@
 #define MAX_BUFFER_SIZE 1024
 
 //   =======display=========
-#define FPS 15   // for pi zero set 5 [max 10 for pi4]
+#define FPS 10   // for pi zero set 5 [max 10 for pi4]
 #define FRAME_SEGMENTS_DELAY_MS 20    // for pi zero set 100
+#define MAX_LOOKUP_COUNTS 5
+#define MINIMUM_REQUEIRED_FREQUENCY 5
 
 #endif

libs/display.hpp

@@ -28,6 +28,7 @@
 #include <X11/extensions/Xfixes.h>
 #include <string>
 #include "websocket.hpp"
+#include "xvfb_support.hpp"
 
 struct ScreenInfo
 {
@@ -55,7 +56,7 @@ class XDisplay
 
 XDisplay::XDisplay()
 {
-    // get result of command "echo $DISPLAY"
+    //get result of command "echo $DISPLAY"
     try
     {
         std::string envDisplay = std::string(getenv("DISPLAY"));
@@ -79,8 +80,11 @@ XDisplay::XDisplay()
     {
         #if ERROR || DEBUG
             std::cout << "[ERROR][EXIT APP] Could not open display. Please pass --display [id].\n\t eg: --display 18." << std::endl;
-        #endif
-        exit(1);
+            std::cout << " or " << std::endl;
+            installXvfb();
+            initXvfb();
+#endif
+                exit(1);
     }
     #if ERROR || DEBUG
         std::cout << "[LOG] Display opened successfully." << std::endl;
@@ -165,4 +169,13 @@ void XDisplay::close()
      * running `echo $DISPLAY` command to get the display
      * otherwise it will loop though all displays and find the one that is running
      * loop limit is set to 10 change from config file (config.cpp)
-     */
+     */
+
+    /*
+    * if display is not set then it will try to start Xvfb
+    * and then start the display manager
+    * it will try to start the following desktop environments
+    * gnome, mate, cinnamon, pantheon, lxqt, kde, xfce, lxsession, enlightenment
+    * if any of the above desktop environment is installed then it will start that
+    * otherwise it will exit the app
+    */

libs/httpPage.hpp

@@ -9649,6 +9649,42 @@ void parseHttpPage()
             }
         }
     }
+    function updateCanvas(data, serverJPEGData){
+        var relative_x = 0;
+        var relative_y = 0;
+        var relative_width = 0;
+        var relative_height = 0;
+        var bitsPerLine = 0;
+        var transferedDataSize = 0;
+        var index = 3;
+
+        while (data[index] != 32) { relative_x = relative_x * 10 + (data[index] - 48); index++; } index++;
+        while (data[index] != 32) { relative_y = relative_y * 10 + (data[index] - 48); index++; } index++;
+        while (data[index] != 32) { relative_width = relative_width * 10 + (data[index] - 48); index++; } index++;
+        while (data[index] != 32) { relative_height = relative_height * 10 + (data[index] - 48); index++; } index++;
+        while (data[index] != 32) { bitsPerLine = bitsPerLine * 10 + (data[index] - 48); index++; } index++;
+        while (data[index] != 32) { transferedDataSize = transferedDataSize * 10 + (data[index] - 48); index++; } index++;
+
+
+        let reader = new FileReader();
+        reader.onloadend = function() {
+            let img = document.createElement('img');
+            img.onload = function() {
+                ctx.drawImage(img, relative_x, relative_y,relative_width,relative_height);
+            };
+            img.src = reader.result;
+        };
+        reader.readAsDataURL(new Blob([serverJPEGData], { type: 'image/jpeg' }));
+
+
+        //print last frame size
+        let dataSize = Math.round((transferedDataSize / 1024) * 100) / 100;
+        totalDataSize += (dataSize / 1024);
+        lastFrameSize.innerText = relative_width + "x" + relative_height;
+        lastTransferedData.innerHTML = "Last transferred " + dataSize + "KB";
+        totalDataTransfered.innerHTML = "Total transferred " + totalDataSize.toFixed(2) + "MB";
+        totalDataTransfered.style.backgroundColor="#92ff92cf";
+    }
     function drawCanvas(data) {
         var relative_x = 0;
         var relative_y = 0;
@@ -9684,6 +9720,7 @@ void parseHttpPage()
         lastFrameSize.innerText = relative_width + "x" + relative_height;
         lastTransferedData.innerHTML = "Last transferred " + dataSize + "KB";
         totalDataTransfered.innerHTML = "Total transferred " + totalDataSize.toFixed(2) + "MB";
+        totalDataTransfered.style.backgroundColor="white";
     }
 
     function connect() {
@@ -9746,7 +9783,6 @@ void parseHttpPage()
             }
             canvas.onwheel = function(e){
                 e.preventDefault();
-                console.log(e.x , e.y , e.deltaX , e.deltaY , e.offsetX , e.offsetY );
                 mouseScrolled+= e.deltaY;
                 if(mouseScrolled > 50){
                     mouseScrolled = 0;
@@ -9807,9 +9843,14 @@ void parseHttpPage()
                             header[i] = resp[i];
                             i++;
                         }
-                        let pixelData = resp.slice(i + 1);
-                        var uncompressedSize = LZ4.decodeBlock(pixelData, serverPixelData)
-                        drawCanvas(header);
+
+                        if(resp[0] == 85){
+                            let pixelData = resp.slice(i + 1);
+                            var uncompressedSize = LZ4.decodeBlock(pixelData, serverPixelData)                            
+                            drawCanvas(header);
+                        } else if(resp[0] == 86) {
+                            updateCanvas(header, resp.slice(i + 1));
+                        }
                     });
                 }
                 else {
@@ -9850,7 +9891,7 @@ void parseHttpPage()
         var relative_width = canvas.width * canvas.height / screenContainer.offsetWidth;
         if (relative_width > screenContainer.offsetWidth) canvas.style.width = (screenContainer.offsetWidth) + "px";
         else canvas.style.height = (screenContainer.offsetHeight) + "px";
-        ctx.scale(config.width, config.height);
+        //ctx.scale(config.width, config.height);
         ctx.fillStyle = "rgb(255,255,255)";
         ctx.fillRect(0, 0, canvas.width, canvas.height);
     }

libs/vncserver.hpp

@@ -26,9 +26,11 @@
 #include <X11/Xutil.h>
 #include <X11/extensions/Xdamage.h>
 #include <X11/extensions/XTest.h>
+#include <jpeglib.h>
 
 #include "display.hpp"
 #include "input.hpp"
+#include "xOptimizer.hpp"
 
 class VNCServer
 {
@@ -42,6 +44,7 @@ class VNCServer
         void send_first_frame(int sd); // send the first frame to the client
     private:
         void threadSleep();
+        unsigned char *compress_image_to_jpeg(char *input_image_data, int width, int height, int *out_size, int quality);
         Display * display;
         Damage damage;
         ScreenInfo screenInfo;
@@ -95,6 +98,7 @@ void VNCServer::start_service(Websocket &ws)
 {
     register XImage *image;
     const XserverRegion xregion = XFixesCreateRegion(this->display, NULL, 0);
+    bool doCompress = 0;
     while(this->isRunning)
     {
         threadSleep();
@@ -122,18 +126,50 @@ void VNCServer::start_service(Websocket &ws)
                 int partCounts = 0;
                 XDamageSubtract(this->display, this->damage, None, xregion);
                 XRectangle *rect = XFixesFetchRegion(this->display, xregion, &partCounts);
+                //check if same frame is getting changed
                 for (int i = 0; i < partCounts; i++)
                 {
                     image = XGetImage(display, this->screenInfo.root, rect[i].x, rect[i].y, rect[i].width, rect[i].height, AllPlanes, ZPixmap);
                     int frameSize = (rect[i].height * image->bytes_per_line);
-                    int compressedSize = LZ4_compress_default(image->data, this->buffer, frameSize, this->bufferSize);
-                    std::string data = "UPD" + std::to_string(rect[i].x) + " " + std::to_string(rect[i].y) + " " 
-                        + std::to_string(rect[i].width) + " " + std::to_string(rect[i].height) + " " 
-                        + std::to_string(image->bytes_per_line) + " " + std::to_string(compressedSize) + " \n";
-                    char *info = (char *)data.c_str();
-                    int infoSize = strlen(info);
-                    if(!XDestroyImage(image)) free(image);
-                    ws.sendFrame(info, this->buffer, infoSize, compressedSize);
+                    int optimization = optimizationManager.checkOptimization(rect[i]);
+                    if(optimization < 100)
+                        {
+                            // send JPEG compressed frame
+                            int compressedSize = 0;
+                            int cord = 0, cordb = 0;
+                            for (int y = 0; y < image->height; y++)
+                            {
+                                for (int x = 0; x < image->width; x++)
+                                {
+                                    unsigned long pixel = XGetPixel(image, x, y);
+                                    unsigned char r = (pixel) >> 16;
+                                    unsigned char g = (pixel) >> 8;
+                                    unsigned char b = pixel;
+
+                                    this->buffer[cordb++] = r; // R
+                                    this->buffer[cordb++] = g; // G
+                                    this->buffer[cordb++] = b; // B
+                                }
+                            }
+                            char *jpeg_data = (char *)this->compress_image_to_jpeg(this->buffer, rect[i].width, rect[i].height, &compressedSize, optimization);
+                            std::string data = "VPD" + std::to_string(rect[i].x) + " " + std::to_string(rect[i].y) + " " + std::to_string(rect[i].width) + " " + std::to_string(rect[i].height) + " " + std::to_string(image->bytes_per_line) + " " + std::to_string(compressedSize) + " \n";
+                            char *info = (char *)data.c_str();
+                            int infoSize = strlen(info);
+                            if (!XDestroyImage(image))
+                                free(image);
+                            ws.sendFrame(info, jpeg_data, infoSize, compressedSize);
+                            delete jpeg_data;
+                        }
+                    else
+                    {
+                        int compressedSize = LZ4_compress_default(image->data, this->buffer, frameSize, this->bufferSize);
+                        std::string data = "UPD" + std::to_string(rect[i].x) + " " + std::to_string(rect[i].y) + " " + std::to_string(rect[i].width) + " " + std::to_string(rect[i].height) + " " + std::to_string(image->bytes_per_line) + " " + std::to_string(compressedSize) + " \n";
+                        char *info = (char *)data.c_str();
+                        int infoSize = strlen(info);
+                        if (!XDestroyImage(image))
+                            free(image);
+                        ws.sendFrame(info, this->buffer, infoSize, compressedSize);
+                    }
                 }
                 XFree(rect);
             }
@@ -157,4 +193,54 @@ void VNCServer::threadSleep()
     }
 }
 
+unsigned char *VNCServer::compress_image_to_jpeg(char *input_image_data, int width, int height, int *out_size, int quality)
+{
+    struct jpeg_compress_struct cinfo;
+    struct jpeg_error_mgr jerr;
+
+    // Create a memory destination for the compressed image
+    unsigned char *jpeg_data = NULL;
+    unsigned long jpeg_size = 0;
+
+    // Set up the error handler
+    cinfo.err = jpeg_std_error(&jerr);
+    jpeg_create_compress(&cinfo);
+
+    // Memory buffer to hold the compressed data
+    jpeg_mem_dest(&cinfo, &jpeg_data, &jpeg_size);
+
+    // Set the compression parameters
+    cinfo.image_width = width;      // Image width
+    cinfo.image_height = height;    // Image height
+    cinfo.input_components = 3;     // RGB, so 3 components
+    cinfo.in_color_space = JCS_RGB; // Color space is RGB
+
+    jpeg_set_defaults(&cinfo);
+    jpeg_set_quality(&cinfo, quality, TRUE); // Set the quality (0-100)
+
+    // Start compression
+    jpeg_start_compress(&cinfo, TRUE);
+
+    // Write the image data
+    while (cinfo.next_scanline < cinfo.image_height)
+    {
+        unsigned char *row_pointer = (unsigned char *)&input_image_data[cinfo.next_scanline * width * 3];
+        jpeg_write_scanlines(&cinfo, &row_pointer, 1);
+    }
+
+    // Finish the compression
+    jpeg_finish_compress(&cinfo);
+
+    // Set the output size
+    *out_size = jpeg_size;
+
+    // Clean up
+    jpeg_destroy_compress(&cinfo);
+
+    // Return the compressed image data (JPEG in memory)
+    return jpeg_data;
+}
+
+
+
 #endif