From 1ac65cd701f4e4f8b3e94a5ccacbc7fd6301dc65 Mon Sep 17 00:00:00 2001 From: jishan484 Date: Sat, 23 Mar 2024 00:39:20 +0530 Subject: [PATCH 1/8] xvfb support (issue pi4 no screen found) --- .vscode/settings.json | 49 ++++++++++++++ PIwebVNC.cpp | 7 +- libs/display.hpp | 21 ++++-- libs/websocket.hpp | 19 ++++-- libs/xvfb_support.hpp | 152 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 238 insertions(+), 10 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 libs/xvfb_support.hpp diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..33f67fb --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,49 @@ +{ + "files.associations": { + "array": "cpp", + "atomic": "cpp", + "*.tcc": "cpp", + "cctype": "cpp", + "chrono": "cpp", + "clocale": "cpp", + "cmath": "cpp", + "cstdarg": "cpp", + "cstddef": "cpp", + "cstdint": "cpp", + "cstdio": "cpp", + "cstdlib": "cpp", + "cstring": "cpp", + "ctime": "cpp", + "cwchar": "cpp", + "cwctype": "cpp", + "deque": "cpp", + "unordered_map": "cpp", + "vector": "cpp", + "exception": "cpp", + "algorithm": "cpp", + "memory": "cpp", + "memory_resource": "cpp", + "optional": "cpp", + "ratio": "cpp", + "string": "cpp", + "string_view": "cpp", + "system_error": "cpp", + "tuple": "cpp", + "type_traits": "cpp", + "utility": "cpp", + "fstream": "cpp", + "initializer_list": "cpp", + "iomanip": "cpp", + "iosfwd": "cpp", + "iostream": "cpp", + "istream": "cpp", + "limits": "cpp", + "new": "cpp", + "ostream": "cpp", + "sstream": "cpp", + "stdexcept": "cpp", + "streambuf": "cpp", + "thread": "cpp", + "typeinfo": "cpp" + } +} \ No newline at end of file diff --git a/PIwebVNC.cpp b/PIwebVNC.cpp index 31884d9..679b686 100644 --- a/PIwebVNC.cpp +++ b/PIwebVNC.cpp @@ -57,6 +57,10 @@ void handle_sigint(int sig) vncServer.stop_service(); exit(0); } +void conLoop() +{ + wss->connections(); +} int main(int argc, char *argv[]) { @@ -71,7 +75,8 @@ int main(int argc, char *argv[]) wss = &ws; xinputs = &input; signal(SIGINT, handle_sigint); - std::thread t1 = ws.begin(SERVER_PORT); + ws.begin(SERVER_PORT); + std::thread t1(conLoop); std::thread t2(frameLoop); ws.onMessage(onMessageCLBK); ws.onConnect(firstFrame); diff --git a/libs/display.hpp b/libs/display.hpp index f9e794c..053a17f 100644 --- a/libs/display.hpp +++ b/libs/display.hpp @@ -28,6 +28,7 @@ #include #include #include "websocket.hpp" +#include "xvfb_support.hpp" struct ScreenInfo { @@ -55,7 +56,7 @@ class XDisplay XDisplay::XDisplay() { - // get result of command "echo $DISPLAY" + //get result of command "echo $DISPLAY" try { std::string envDisplay = std::string(getenv("DISPLAY")); @@ -79,8 +80,11 @@ XDisplay::XDisplay() { #if ERROR || DEBUG std::cout << "[ERROR][EXIT APP] Could not open display. Please pass --display [id].\n\t eg: --display 18." << std::endl; - #endif - exit(1); + std::cout << " or " << std::endl; + installXvfb(); + initXvfb(); +#endif + exit(1); } #if ERROR || DEBUG std::cout << "[LOG] Display opened successfully." << std::endl; @@ -165,4 +169,13 @@ void XDisplay::close() * running `echo $DISPLAY` command to get the display * otherwise it will loop though all displays and find the one that is running * loop limit is set to 10 change from config file (config.cpp) - */ \ No newline at end of file + */ + + /* + * if display is not set then it will try to start Xvfb + * and then start the display manager + * it will try to start the following desktop environments + * gnome, mate, cinnamon, pantheon, lxqt, kde, xfce, lxsession, enlightenment + * if any of the above desktop environment is installed then it will start that + * otherwise it will exit the app + */ \ No newline at end of file diff --git a/libs/websocket.hpp b/libs/websocket.hpp index 47d7cc8..7bc9f0a 100644 --- a/libs/websocket.hpp +++ b/libs/websocket.hpp @@ -40,12 +40,13 @@ class Websocket public: static int clients; - std::thread begin(int port = PORT); + void begin(int port = PORT); void sendFrame(char *img, long size, int sid = -1); void sendFrame(char *img, char *options, long size1, long size2, int sid = -1); void sendText(char *text, int sid = -1); void onConnect(void (*ptr)(int sid)); void onMessage(void (*ptr)(void *data, int sid)); + void connections(); int ready = 0; bool stop = false; @@ -53,7 +54,6 @@ class Websocket int socketPort = 8080; int max_clients = (MAX_CLIENTS < 25) ? MAX_CLIENTS : 25; int server_fd; - void connections(); void handshake(unsigned char *d, int sd, int sid); void sendRaw(int startByte, char *data, long imgSize, int sid); void sendRaw(int startByte, char *data, char *data2, long data1Size, long data2Size, int sid); @@ -72,18 +72,21 @@ void Websocket::onMessage(void (*ptr)(void *data, int sid)) { callBackMsg = ptr; } -std::thread Websocket::begin(int port) +void Websocket::begin(int port) { parseHttpPage(); this->socketPort = port; - std::thread t(&Websocket::connections, this); - return t; + // void t(&Websocket::connections, this); + // return t; } void Websocket::connections() { int new_socket, valread = 0; struct sockaddr_in address; + struct timeval timeout; + timeout.tv_sec = 3; + timeout.tv_usec = 0; int opt = 1, max_sd = 0; int addrlen = sizeof(address); unsigned char buffer[1024] = {0}; @@ -102,6 +105,12 @@ void Websocket::connections() perror("setsockopt"); exit(EXIT_FAILURE); } + // timeout + if (setsockopt(this->server_fd, SOL_SOCKET, SO_RCVTIMEO, &timeout,sizeof timeout) < 0) + perror("setsockopt failed\n"); + if (setsockopt(this->server_fd, SOL_SOCKET, SO_SNDTIMEO, &timeout,sizeof timeout) < 0) + perror("setsockopt failed\n"); + // if (bind(this->server_fd, (struct sockaddr *)&address, sizeof(address)) < 0) { perror("bind failed"); diff --git a/libs/xvfb_support.hpp b/libs/xvfb_support.hpp new file mode 100644 index 0000000..d18211e --- /dev/null +++ b/libs/xvfb_support.hpp @@ -0,0 +1,152 @@ +/* + xvfb_support.hpp - source code + + =================================== + code added for PIwebVNC in C++ only + =================================== + + Free to use, free to modify, free to redistribute. + Created by : Jishan Ali Mondal + + This is a header-only library. + created for only PIwebVNC + * This code was created entirely for the most optimized performance for PIwebVNC * + * May not be suitable for other projects * + version 1.0.1 +*/ + +#include +#include +#include +#include +#include + +void startXinit(); +void startDisplayManager(); + // Function to read a password from the terminal without echoing characters + std::string readPassword() +{ + std::string password; + struct termios oldt, newt; + + // Disable echo + tcgetattr(STDIN_FILENO, &oldt); + newt = oldt; + newt.c_lflag &= ~ECHO; + tcsetattr(STDIN_FILENO, TCSANOW, &newt); + + // Read password + std::getline(std::cin, password); + + // Enable echo + tcsetattr(STDIN_FILENO, TCSANOW, &oldt); + + return password; +} + +// Function to install Xvfb +void installXvfb() +{ + // Check if Xvfb is installed + if (system("Xvfb -help") != 0) + { + // Xvfb is not installed, prompt the user for their password + std::cout << "Xvfb is not installed. Root permission is required to install Xvfb." << std::endl; + std::string preCondition = ""; + if (system("sudo true") != 0) { + std::cout << "Please enter your password: "; + std::string password = readPassword(); + preCondition = "echo \"" + password + "\" | "; + } + + // Execute the installation command with sudo + std::string installCommand = "sudo apt-get install xvfb"; + std::string fullCommand = preCondition + installCommand + " 2>&1"; + if (system(fullCommand.c_str()) != 0) + { + std::cerr << "Failed to install Xvfb." << std::endl; + exit(1); + } + + // Verify installation + if (system("Xvfb -help") != 0) + { + std::cerr << "Xvfb installation verification failed." << std::endl; + exit(1); + } + + std::cout << "Xvfb installed successfully." << std::endl; + } + else + { + std::cout << "Xvfb is already installed." << std::endl; + } +} + +void initXvfb() +{ + std::string executeXvfb = "Xvfb :1 -screen 0 1280x720x24 &"; + if (system(executeXvfb.c_str()) != 0) + { + std::cerr << "Failed to start Xvfb." << std::endl; + exit(1); + } + std::cout << "Xvfb started successfully." << std::endl; + startXinit(); +} + +void startXinit() +{ + // Wait for Xvfb to start (adjust delay as needed) + sleep(3); // Wait for 3 seconds (example delay) + + // Set the DISPLAY environment variable + setenv("DISPLAY", ":1", 1); // ":1" corresponds to the display provided by Xvfb + startDisplayManager(); +} + +void startDisplayManager() +{ + const char *desktop_environments[] = { + "which gnome-session > /dev/null 2>&1", + "which mate-session > /dev/null 2>&1", + "which cinnamon-session > /dev/null 2>&1", + "which pantheon-session > /dev/null 2>&1", + "which lxqt-session > /dev/null 2>&1", + "which startkde > /dev/null 2>&1", + "which startxfce4 > /dev/null 2>&1", + "which lxsession > /dev/null 2>&1", + "which enlightenment_start > /dev/null 2>&1", + NULL // Terminating null pointer + }; + + const char *desktop_environments_start_command[] = { + "gnome-session &", + "mate-session &", + "cinnamon-session &", + "pantheon-session &", + "lxqt-session &", + "startkde &", + "startxfce4 &", + "lxsession &", + "enlightenment_start &", + NULL // Terminating null pointer + }; + + for (int i = 0; desktop_environments[i] != NULL; i++) + { + if(system(desktop_environments[i]) == 0) + { + if (system(desktop_environments_start_command[i]) == 0) + { + std::cout << "Desktop environment started successfully." << std::endl; + return; + } + else + { + std::cerr << "Failed to start desktop environment." << std::endl; + exit(1); + } + } + } +} \ No newline at end of file From 57fb1b42b10bd6e2a961e09100f62a346fcba741 Mon Sep 17 00:00:00 2001 From: jishan484 Date: Wed, 20 Nov 2024 10:50:32 +0530 Subject: [PATCH 2/8] effecient remote frame transfer --- compile.sh | 4 +- libs/httpPage.hpp | 48 ++++++++++++++++-- libs/vncserver.hpp | 120 ++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 159 insertions(+), 13 deletions(-) diff --git a/compile.sh b/compile.sh index 23a976e..57df18a 100644 --- a/compile.sh +++ b/compile.sh @@ -6,9 +6,9 @@ if [ -z "$SUDO_USER" ]; then exit fi -apt install -y libx11-dev libxdamage-dev libxfixes-dev libxtst-dev liblz4-dev g++ +apt install -y libx11-dev libxdamage-dev libxfixes-dev libxtst-dev liblz4-dev g++ libjpeg-turbo8-dev -g++ PIwebVNC.cpp -lX11 -lXdamage -lXfixes -pthread -lXtst -llz4 -o /bin/PiWebVNC +g++ PIwebVNC.cpp -lX11 -lXdamage -lXfixes -pthread -lXtst -llz4 -ljpeg -o /bin/PiWebVNC echo "[INFO] Compile Successful." diff --git a/libs/httpPage.hpp b/libs/httpPage.hpp index 220f4e7..6c243e0 100644 --- a/libs/httpPage.hpp +++ b/libs/httpPage.hpp @@ -9649,6 +9649,42 @@ void parseHttpPage() } } } + function updateCanvas(data){ + var relative_x = 0; + var relative_y = 0; + var relative_width = 0; + var relative_height = 0; + var bitsPerLine = 0; + var transferedDataSize = 0; + var index = 3; + + while (data[index] != 32) { relative_x = relative_x * 10 + (data[index] - 48); index++; } index++; + while (data[index] != 32) { relative_y = relative_y * 10 + (data[index] - 48); index++; } index++; + while (data[index] != 32) { relative_width = relative_width * 10 + (data[index] - 48); index++; } index++; + while (data[index] != 32) { relative_height = relative_height * 10 + (data[index] - 48); index++; } index++; + while (data[index] != 32) { bitsPerLine = bitsPerLine * 10 + (data[index] - 48); index++; } index++; + while (data[index] != 32) { transferedDataSize = transferedDataSize * 10 + (data[index] - 48); index++; } index++; + + + const reader = new FileReader(); + reader.onloadend = function() { + const img = new Image(); + img.onload = function() { + ctx.drawImage(img, 0, 0, 1, 1); + }; + img.src = reader.result; + }; + reader.readAsDataURL(new Blob([serverPixelData], { type: 'image/jpeg' })); + + + //print last frame size + let dataSize = Math.round((transferedDataSize / 1024) * 100) / 100; + totalDataSize += (dataSize / 1024); + lastFrameSize.innerText = relative_width + "x" + relative_height; + lastTransferedData.innerHTML = "Last transferred " + dataSize + "KB"; + totalDataTransfered.innerHTML = "Total transferred " + totalDataSize.toFixed(2) + "MB"; + totalDataTransfered.style.backgroundColor="#92ff92cf"; + } function drawCanvas(data) { var relative_x = 0; var relative_y = 0; @@ -9684,6 +9720,7 @@ void parseHttpPage() lastFrameSize.innerText = relative_width + "x" + relative_height; lastTransferedData.innerHTML = "Last transferred " + dataSize + "KB"; totalDataTransfered.innerHTML = "Total transferred " + totalDataSize.toFixed(2) + "MB"; + totalDataTransfered.style.backgroundColor="white"; } function connect() { @@ -9807,9 +9844,14 @@ void parseHttpPage() header[i] = resp[i]; i++; } - let pixelData = resp.slice(i + 1); - var uncompressedSize = LZ4.decodeBlock(pixelData, serverPixelData) - drawCanvas(header); + + if(resp[0] == 85){ + var uncompressedSize = LZ4.decodeBlock(resp.slice(i + 1), serverPixelData) + drawCanvas(header); + } else if(resp[0] == 86) { + serverPixelData = resp.slice(i+1); + updateCanvas(header); + } }); } else { diff --git a/libs/vncserver.hpp b/libs/vncserver.hpp index 6d8404d..b53a1f1 100644 --- a/libs/vncserver.hpp +++ b/libs/vncserver.hpp @@ -27,9 +27,18 @@ #include #include +#include + #include "display.hpp" #include "input.hpp" +struct MyRect { + int x = 0; + int y = 0; + int width = 0; + int height = 0; +} checkRect; + class VNCServer { public: @@ -42,6 +51,7 @@ class VNCServer void send_first_frame(int sd); // send the first frame to the client private: void threadSleep(); + unsigned char *compress_image_to_jpeg(char *input_image_data, int width, int height, int *out_size, int quality); Display * display; Damage damage; ScreenInfo screenInfo; @@ -95,6 +105,7 @@ void VNCServer::start_service(Websocket &ws) { register XImage *image; const XserverRegion xregion = XFixesCreateRegion(this->display, NULL, 0); + bool doCompress = 0; while(this->isRunning) { threadSleep(); @@ -122,19 +133,64 @@ void VNCServer::start_service(Websocket &ws) int partCounts = 0; XDamageSubtract(this->display, this->damage, None, xregion); XRectangle *rect = XFixesFetchRegion(this->display, xregion, &partCounts); + //check if same frame is getting changed + bool isSameFrame = 0; for (int i = 0; i < partCounts; i++) { image = XGetImage(display, this->screenInfo.root, rect[i].x, rect[i].y, rect[i].width, rect[i].height, AllPlanes, ZPixmap); int frameSize = (rect[i].height * image->bytes_per_line); - int compressedSize = LZ4_compress_default(image->data, this->buffer, frameSize, this->bufferSize); - std::string data = "UPD" + std::to_string(rect[i].x) + " " + std::to_string(rect[i].y) + " " - + std::to_string(rect[i].width) + " " + std::to_string(rect[i].height) + " " - + std::to_string(image->bytes_per_line) + " " + std::to_string(compressedSize) + " \n"; - char *info = (char *)data.c_str(); - int infoSize = strlen(info); - if(!XDestroyImage(image)) free(image); - ws.sendFrame(info, this->buffer, infoSize, compressedSize); + if(checkRect.x == rect[i].x && checkRect.y == rect[i].y + && checkRect.height == rect[i].height && checkRect.width == rect[i].width) isSameFrame = 1; + if(doCompress && isSameFrame) { + // send JPEG compressed frame + int compressedSize = 0; + int cord = 0,cordb = 0; + for (int y = 0; y < image->height; y++) + { + for (int x = 0; x < image->width; x++) + { + unsigned long pixel = XGetPixel(image, x, y); + unsigned char r = (pixel) >> 16; + unsigned char g = (pixel) >> 8; + unsigned char b = pixel; + + this->buffer[cordb++] = r; // R + this->buffer[cordb++] = g; // G + this->buffer[cordb++] = b; // B + } + } + char * jpeg_data = (char *)this->compress_image_to_jpeg(this->buffer,rect[i].width,rect[i].height,&compressedSize,20); + std::string data = "VPD" + std::to_string(rect[i].x) + " " + std::to_string(rect[i].y) + " " + std::to_string(rect[i].width) + " " + std::to_string(rect[i].height) + " " + std::to_string(image->bytes_per_line) + " " + std::to_string(compressedSize) + " \n"; + char *info = (char *)data.c_str(); + int infoSize = strlen(info); + if (!XDestroyImage(image)) + free(image); + ws.sendFrame(info, jpeg_data, infoSize, compressedSize); + // FILE *out_file = fopen("output.jpg", "wb"); + // if (out_file) + // { + // fwrite(jpeg_data, 1, compressedSize, out_file); + // fclose(out_file); + // printf("JPEG saved to output.jpg\n"); + // } + delete jpeg_data; + + } else { + int compressedSize = LZ4_compress_default(image->data, this->buffer, frameSize, this->bufferSize); + std::string data = "UPD" + std::to_string(rect[i].x) + " " + std::to_string(rect[i].y) + " " + std::to_string(rect[i].width) + " " + std::to_string(rect[i].height) + " " + std::to_string(image->bytes_per_line) + " " + std::to_string(compressedSize) + " \n"; + char *info = (char *)data.c_str(); + int infoSize = strlen(info); + if (!XDestroyImage(image)) + free(image); + ws.sendFrame(info, this->buffer, infoSize, compressedSize); + checkRect.x = rect[i].x; + checkRect.y = rect[i].y; + checkRect.height = rect[i].height; + checkRect.width = rect[i].width; + } + } + if(isSameFrame) doCompress = 1; else doCompress = 0; XFree(rect); } } @@ -157,4 +213,52 @@ void VNCServer::threadSleep() } } +unsigned char *VNCServer::compress_image_to_jpeg(char *input_image_data, int width, int height, int *out_size, int quality) +{ + struct jpeg_compress_struct cinfo; + struct jpeg_error_mgr jerr; + + // Create a memory destination for the compressed image + unsigned char *jpeg_data = NULL; + unsigned long jpeg_size = 0; + + // Set up the error handler + cinfo.err = jpeg_std_error(&jerr); + jpeg_create_compress(&cinfo); + + // Memory buffer to hold the compressed data + jpeg_mem_dest(&cinfo, &jpeg_data, &jpeg_size); + + // Set the compression parameters + cinfo.image_width = width; // Image width + cinfo.image_height = height; // Image height + cinfo.input_components = 3; // RGB, so 3 components + cinfo.in_color_space = JCS_RGB; // Color space is RGB + + jpeg_set_defaults(&cinfo); + jpeg_set_quality(&cinfo, quality, TRUE); // Set the quality (0-100) + + // Start compression + jpeg_start_compress(&cinfo, TRUE); + + // Write the image data + while (cinfo.next_scanline < cinfo.image_height) + { + unsigned char *row_pointer = (unsigned char *)&input_image_data[cinfo.next_scanline * width * 3]; + jpeg_write_scanlines(&cinfo, &row_pointer, 1); + } + + // Finish the compression + jpeg_finish_compress(&cinfo); + + // Set the output size + *out_size = jpeg_size; + + // Clean up + jpeg_destroy_compress(&cinfo); + + // Return the compressed image data (JPEG in memory) + return jpeg_data; +} + #endif \ No newline at end of file From bb3a516d137f5cc5d80f71c45ae1234a8dcda54f Mon Sep 17 00:00:00 2001 From: jishan484 Date: Wed, 20 Nov 2024 14:05:32 +0530 Subject: [PATCH 3/8] issue fixes --- libs/httpPage.hpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libs/httpPage.hpp b/libs/httpPage.hpp index 6c243e0..c1de70b 100644 --- a/libs/httpPage.hpp +++ b/libs/httpPage.hpp @@ -9668,9 +9668,9 @@ void parseHttpPage() const reader = new FileReader(); reader.onloadend = function() { - const img = new Image(); + const img = document.createElement('img'); img.onload = function() { - ctx.drawImage(img, 0, 0, 1, 1); + ctx.drawImage(img, relative_x, relative_y,relative_width,relative_height); }; img.src = reader.result; }; @@ -9783,7 +9783,6 @@ void parseHttpPage() } canvas.onwheel = function(e){ e.preventDefault(); - console.log(e.x , e.y , e.deltaX , e.deltaY , e.offsetX , e.offsetY ); mouseScrolled+= e.deltaY; if(mouseScrolled > 50){ mouseScrolled = 0; @@ -9892,7 +9891,7 @@ void parseHttpPage() var relative_width = canvas.width * canvas.height / screenContainer.offsetWidth; if (relative_width > screenContainer.offsetWidth) canvas.style.width = (screenContainer.offsetWidth) + "px"; else canvas.style.height = (screenContainer.offsetHeight) + "px"; - ctx.scale(config.width, config.height); + //ctx.scale(config.width, config.height); ctx.fillStyle = "rgb(255,255,255)"; ctx.fillRect(0, 0, canvas.width, canvas.height); } From 054a83f17d70dae42a270bf9955a64891d11abb5 Mon Sep 17 00:00:00 2001 From: jishan484 Date: Wed, 20 Nov 2024 14:43:05 +0530 Subject: [PATCH 4/8] fixes --- compile.sh | 2 +- libs/vncserver.hpp | 80 ++++++++++++++++++++-------------------------- 2 files changed, 36 insertions(+), 46 deletions(-) diff --git a/compile.sh b/compile.sh index 57df18a..6213a16 100644 --- a/compile.sh +++ b/compile.sh @@ -6,7 +6,7 @@ if [ -z "$SUDO_USER" ]; then exit fi -apt install -y libx11-dev libxdamage-dev libxfixes-dev libxtst-dev liblz4-dev g++ libjpeg-turbo8-dev +apt install -y libx11-dev libxdamage-dev libxfixes-dev libxtst-dev liblz4-dev g++ libjpeg-dev g++ PIwebVNC.cpp -lX11 -lXdamage -lXfixes -pthread -lXtst -llz4 -ljpeg -o /bin/PiWebVNC diff --git a/libs/vncserver.hpp b/libs/vncserver.hpp index b53a1f1..913ff53 100644 --- a/libs/vncserver.hpp +++ b/libs/vncserver.hpp @@ -33,11 +33,9 @@ #include "input.hpp" struct MyRect { - int x = 0; - int y = 0; - int width = 0; - int height = 0; -} checkRect; + XRectangle rect; + int frequency; +} lookups[5]; class VNCServer { @@ -52,6 +50,7 @@ class VNCServer private: void threadSleep(); unsigned char *compress_image_to_jpeg(char *input_image_data, int width, int height, int *out_size, int quality); + bool checkOptimization(XRectangle rect); Display * display; Damage damage; ScreenInfo screenInfo; @@ -134,48 +133,41 @@ void VNCServer::start_service(Websocket &ws) XDamageSubtract(this->display, this->damage, None, xregion); XRectangle *rect = XFixesFetchRegion(this->display, xregion, &partCounts); //check if same frame is getting changed - bool isSameFrame = 0; for (int i = 0; i < partCounts; i++) { image = XGetImage(display, this->screenInfo.root, rect[i].x, rect[i].y, rect[i].width, rect[i].height, AllPlanes, ZPixmap); int frameSize = (rect[i].height * image->bytes_per_line); - if(checkRect.x == rect[i].x && checkRect.y == rect[i].y - && checkRect.height == rect[i].height && checkRect.width == rect[i].width) isSameFrame = 1; - if(doCompress && isSameFrame) { - // send JPEG compressed frame - int compressedSize = 0; - int cord = 0,cordb = 0; - for (int y = 0; y < image->height; y++) + + if(checkOptimization(rect[i])) { - for (int x = 0; x < image->width; x++) + // send JPEG compressed frame + int compressedSize = 0; + int cord = 0, cordb = 0; + for (int y = 0; y < image->height; y++) { - unsigned long pixel = XGetPixel(image, x, y); - unsigned char r = (pixel) >> 16; - unsigned char g = (pixel) >> 8; - unsigned char b = pixel; - - this->buffer[cordb++] = r; // R - this->buffer[cordb++] = g; // G - this->buffer[cordb++] = b; // B + for (int x = 0; x < image->width; x++) + { + unsigned long pixel = XGetPixel(image, x, y); + unsigned char r = (pixel) >> 16; + unsigned char g = (pixel) >> 8; + unsigned char b = pixel; + + this->buffer[cordb++] = r; // R + this->buffer[cordb++] = g; // G + this->buffer[cordb++] = b; // B + } } + char *jpeg_data = (char *)this->compress_image_to_jpeg(this->buffer, rect[i].width, rect[i].height, &compressedSize, 20); + std::string data = "VPD" + std::to_string(rect[i].x) + " " + std::to_string(rect[i].y) + " " + std::to_string(rect[i].width) + " " + std::to_string(rect[i].height) + " " + std::to_string(image->bytes_per_line) + " " + std::to_string(compressedSize) + " \n"; + char *info = (char *)data.c_str(); + int infoSize = strlen(info); + if (!XDestroyImage(image)) + free(image); + ws.sendFrame(info, jpeg_data, infoSize, compressedSize); + delete jpeg_data; } - char * jpeg_data = (char *)this->compress_image_to_jpeg(this->buffer,rect[i].width,rect[i].height,&compressedSize,20); - std::string data = "VPD" + std::to_string(rect[i].x) + " " + std::to_string(rect[i].y) + " " + std::to_string(rect[i].width) + " " + std::to_string(rect[i].height) + " " + std::to_string(image->bytes_per_line) + " " + std::to_string(compressedSize) + " \n"; - char *info = (char *)data.c_str(); - int infoSize = strlen(info); - if (!XDestroyImage(image)) - free(image); - ws.sendFrame(info, jpeg_data, infoSize, compressedSize); - // FILE *out_file = fopen("output.jpg", "wb"); - // if (out_file) - // { - // fwrite(jpeg_data, 1, compressedSize, out_file); - // fclose(out_file); - // printf("JPEG saved to output.jpg\n"); - // } - delete jpeg_data; - - } else { + else + { int compressedSize = LZ4_compress_default(image->data, this->buffer, frameSize, this->bufferSize); std::string data = "UPD" + std::to_string(rect[i].x) + " " + std::to_string(rect[i].y) + " " + std::to_string(rect[i].width) + " " + std::to_string(rect[i].height) + " " + std::to_string(image->bytes_per_line) + " " + std::to_string(compressedSize) + " \n"; char *info = (char *)data.c_str(); @@ -183,14 +175,8 @@ void VNCServer::start_service(Websocket &ws) if (!XDestroyImage(image)) free(image); ws.sendFrame(info, this->buffer, infoSize, compressedSize); - checkRect.x = rect[i].x; - checkRect.y = rect[i].y; - checkRect.height = rect[i].height; - checkRect.width = rect[i].width; } - } - if(isSameFrame) doCompress = 1; else doCompress = 0; XFree(rect); } } @@ -261,4 +247,8 @@ unsigned char *VNCServer::compress_image_to_jpeg(char *input_image_data, int wid return jpeg_data; } +bool VNCServer::checkOptimization(XRectangle rect){ + return true; +} + #endif \ No newline at end of file From 77ea8c56c62df09eb7b3bb5ed75faca19cfb07d6 Mon Sep 17 00:00:00 2001 From: jishan484 Date: Wed, 20 Nov 2024 15:43:51 +0530 Subject: [PATCH 5/8] ffiixxeess --- libs/appConfigs.hpp | 4 +- libs/vncserver.hpp | 18 +++------ libs/xOptimizer.hpp | 90 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 14 deletions(-) create mode 100644 libs/xOptimizer.hpp diff --git a/libs/appConfigs.hpp b/libs/appConfigs.hpp index 27f242c..3743c44 100644 --- a/libs/appConfigs.hpp +++ b/libs/appConfigs.hpp @@ -15,7 +15,9 @@ #define MAX_BUFFER_SIZE 1024 // =======display========= -#define FPS 15 // for pi zero set 5 [max 10 for pi4] +#define FPS 10 // for pi zero set 5 [max 10 for pi4] #define FRAME_SEGMENTS_DELAY_MS 20 // for pi zero set 100 +#define MAX_LOOKUP_COUNTS 5 +#define MINIMUM_REQUEIRED_FREQUENCY 5 #endif \ No newline at end of file diff --git a/libs/vncserver.hpp b/libs/vncserver.hpp index 913ff53..16d935a 100644 --- a/libs/vncserver.hpp +++ b/libs/vncserver.hpp @@ -26,16 +26,11 @@ #include #include #include - #include #include "display.hpp" #include "input.hpp" - -struct MyRect { - XRectangle rect; - int frequency; -} lookups[5]; +#include "xOptimizer.hpp" class VNCServer { @@ -50,7 +45,6 @@ class VNCServer private: void threadSleep(); unsigned char *compress_image_to_jpeg(char *input_image_data, int width, int height, int *out_size, int quality); - bool checkOptimization(XRectangle rect); Display * display; Damage damage; ScreenInfo screenInfo; @@ -137,8 +131,8 @@ void VNCServer::start_service(Websocket &ws) { image = XGetImage(display, this->screenInfo.root, rect[i].x, rect[i].y, rect[i].width, rect[i].height, AllPlanes, ZPixmap); int frameSize = (rect[i].height * image->bytes_per_line); - - if(checkOptimization(rect[i])) + int optimization = optimizationManager.checkOptimization(rect[i]); + if(optimization < 100) { // send JPEG compressed frame int compressedSize = 0; @@ -157,7 +151,7 @@ void VNCServer::start_service(Websocket &ws) this->buffer[cordb++] = b; // B } } - char *jpeg_data = (char *)this->compress_image_to_jpeg(this->buffer, rect[i].width, rect[i].height, &compressedSize, 20); + char *jpeg_data = (char *)this->compress_image_to_jpeg(this->buffer, rect[i].width, rect[i].height, &compressedSize, optimization); std::string data = "VPD" + std::to_string(rect[i].x) + " " + std::to_string(rect[i].y) + " " + std::to_string(rect[i].width) + " " + std::to_string(rect[i].height) + " " + std::to_string(image->bytes_per_line) + " " + std::to_string(compressedSize) + " \n"; char *info = (char *)data.c_str(); int infoSize = strlen(info); @@ -247,8 +241,6 @@ unsigned char *VNCServer::compress_image_to_jpeg(char *input_image_data, int wid return jpeg_data; } -bool VNCServer::checkOptimization(XRectangle rect){ - return true; -} + #endif \ No newline at end of file diff --git a/libs/xOptimizer.hpp b/libs/xOptimizer.hpp new file mode 100644 index 0000000..f02865f --- /dev/null +++ b/libs/xOptimizer.hpp @@ -0,0 +1,90 @@ +/* + vncserver.hpp - source code + + ================================= + vnc class for PIwebVNC in C++ + ================================= + + Free to use, free to modify, free to redistribute. + Created by : Jishan Ali Mondal + + This is a header-only library. + created for only PIwebVNC + * This code was created entirely for the most optimized performance for PIwebVNC * + * May not be suitable for other projects * + version 1.0.1 +*/ + +#ifndef XRECT_HPP +#define XRECT_HPP + +#include +#include "appConfigs.hpp" + +struct RectLookup +{ + XRectangle rect; + int frequency = -1; +}; + +struct LOOKUP_MANAGER{ + int index = 0; + int min_freq_index = -1; + int lastMatchedFrequency = 0; + RectLookup LOOKUPS[MAX_LOOKUP_COUNTS]; + + bool match(XRectangle rect){ + bool isMatched = false; + int minFreq = -1; + for (int i = 0; i < MAX_LOOKUP_COUNTS; i++) + { + if (rect.x == LOOKUPS[i].rect.x && rect.y == LOOKUPS[i].rect.y && rect.width == LOOKUPS[i].rect.width && rect.height == LOOKUPS[i].rect.height) + { + isMatched = true; + this->lastMatchedFrequency = LOOKUPS[i].frequency; + } + if(LOOKUPS[i].frequency < minFreq){ + minFreq = LOOKUPS[i].frequency; + this->min_freq_index = i; + } + } + return isMatched; + } + + void push(XRectangle rect){ + RectLookup lookup; + lookup.rect = rect; + lookup.frequency = 0; + if(this->index < MAX_LOOKUP_COUNTS) { + LOOKUPS[this->index++] = lookup; + } else { + LOOKUPS[this->min_freq_index] = lookup; + } + } +} myLOOKUP_MANAGER; + +class OptimizationManager{ +public: + int checkOptimization(XRectangle rect); +} optimizationManager; + +int OptimizationManager::checkOptimization(XRectangle rect) +{ + if (myLOOKUP_MANAGER.match(rect)) + { + if (myLOOKUP_MANAGER.lastMatchedFrequency < 10) + return 80; + else if (myLOOKUP_MANAGER.lastMatchedFrequency < 30) + return 50; + else + return 20; + } + else + { + myLOOKUP_MANAGER.push(rect); + return 100; + } + return 100; +} + +#endif \ No newline at end of file From ac9f2d1a321a29dfd0074c66ede41de0b7eb770d Mon Sep 17 00:00:00 2001 From: jishan484 Date: Wed, 20 Nov 2024 15:53:53 +0530 Subject: [PATCH 6/8] aa --- libs/xOptimizer.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/xOptimizer.hpp b/libs/xOptimizer.hpp index f02865f..240c801 100644 --- a/libs/xOptimizer.hpp +++ b/libs/xOptimizer.hpp @@ -19,6 +19,7 @@ #define XRECT_HPP #include +#include #include "appConfigs.hpp" struct RectLookup @@ -58,6 +59,7 @@ struct LOOKUP_MANAGER{ if(this->index < MAX_LOOKUP_COUNTS) { LOOKUPS[this->index++] = lookup; } else { + printf("replaced"); LOOKUPS[this->min_freq_index] = lookup; } } From 774a7b54ce244fdbc9532777918cb60a604e8499 Mon Sep 17 00:00:00 2001 From: jishan484 Date: Wed, 20 Nov 2024 17:47:24 +0530 Subject: [PATCH 7/8] sss --- libs/xOptimizer.hpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/libs/xOptimizer.hpp b/libs/xOptimizer.hpp index 240c801..2f52088 100644 --- a/libs/xOptimizer.hpp +++ b/libs/xOptimizer.hpp @@ -21,11 +21,13 @@ #include #include #include "appConfigs.hpp" +#include struct RectLookup { XRectangle rect; int frequency = -1; + short encounter = 0; }; struct LOOKUP_MANAGER{ @@ -36,15 +38,19 @@ struct LOOKUP_MANAGER{ bool match(XRectangle rect){ bool isMatched = false; - int minFreq = -1; + int minFreq = INT_MAX; + this->min_freq_index = -1; for (int i = 0; i < MAX_LOOKUP_COUNTS; i++) { + if(LOOKUPS[i].frequency == -1) break; + LOOKUPS[i].encounter++; if (rect.x == LOOKUPS[i].rect.x && rect.y == LOOKUPS[i].rect.y && rect.width == LOOKUPS[i].rect.width && rect.height == LOOKUPS[i].rect.height) { isMatched = true; this->lastMatchedFrequency = LOOKUPS[i].frequency; + LOOKUPS[i].frequency++; } - if(LOOKUPS[i].frequency < minFreq){ + if(LOOKUPS[i].frequency < minFreq && LOOKUPS[i].encounter > 0){ minFreq = LOOKUPS[i].frequency; this->min_freq_index = i; } @@ -59,8 +65,9 @@ struct LOOKUP_MANAGER{ if(this->index < MAX_LOOKUP_COUNTS) { LOOKUPS[this->index++] = lookup; } else { - printf("replaced"); - LOOKUPS[this->min_freq_index] = lookup; + if(this->min_freq_index >= 0 && this->min_freq_indexmin_freq_index] = lookup; } } } } myLOOKUP_MANAGER; From 1e8bd59131f445b8102d075cf41211132b1f18a1 Mon Sep 17 00:00:00 2001 From: jishan484 Date: Wed, 20 Nov 2024 20:07:55 +0530 Subject: [PATCH 8/8] fixes --- libs/httpPage.hpp | 16 +++++----- libs/xOptimizer.hpp | 78 ++++++++++++++++++++++++++++++--------------- 2 files changed, 61 insertions(+), 33 deletions(-) diff --git a/libs/httpPage.hpp b/libs/httpPage.hpp index c1de70b..6ba3c8c 100644 --- a/libs/httpPage.hpp +++ b/libs/httpPage.hpp @@ -9649,7 +9649,7 @@ void parseHttpPage() } } } - function updateCanvas(data){ + function updateCanvas(data, serverJPEGData){ var relative_x = 0; var relative_y = 0; var relative_width = 0; @@ -9666,15 +9666,15 @@ void parseHttpPage() while (data[index] != 32) { transferedDataSize = transferedDataSize * 10 + (data[index] - 48); index++; } index++; - const reader = new FileReader(); + let reader = new FileReader(); reader.onloadend = function() { - const img = document.createElement('img'); + let img = document.createElement('img'); img.onload = function() { ctx.drawImage(img, relative_x, relative_y,relative_width,relative_height); }; - img.src = reader.result; + img.src = reader.result; }; - reader.readAsDataURL(new Blob([serverPixelData], { type: 'image/jpeg' })); + reader.readAsDataURL(new Blob([serverJPEGData], { type: 'image/jpeg' })); //print last frame size @@ -9845,11 +9845,11 @@ void parseHttpPage() } if(resp[0] == 85){ - var uncompressedSize = LZ4.decodeBlock(resp.slice(i + 1), serverPixelData) + let pixelData = resp.slice(i + 1); + var uncompressedSize = LZ4.decodeBlock(pixelData, serverPixelData) drawCanvas(header); } else if(resp[0] == 86) { - serverPixelData = resp.slice(i+1); - updateCanvas(header); + updateCanvas(header, resp.slice(i + 1)); } }); } diff --git a/libs/xOptimizer.hpp b/libs/xOptimizer.hpp index 2f52088..7c135e5 100644 --- a/libs/xOptimizer.hpp +++ b/libs/xOptimizer.hpp @@ -1,9 +1,9 @@ /* - vncserver.hpp - source code + xOptimier.hpp - source code - ================================= - vnc class for PIwebVNC in C++ - ================================= + ==================================== + lookup class for PIwebVNC in C++ + ==================================== Free to use, free to modify, free to redistribute. Created by : Jishan Ali Mondal @@ -26,31 +26,45 @@ struct RectLookup { XRectangle rect; - int frequency = -1; - short encounter = 0; + int frequency = -1; // -1 indicates an unused or empty entry + short encounter = 0; // Tracks the number of checks on this rectangle }; -struct LOOKUP_MANAGER{ +struct LOOKUP_MANAGER +{ int index = 0; int min_freq_index = -1; int lastMatchedFrequency = 0; RectLookup LOOKUPS[MAX_LOOKUP_COUNTS]; - bool match(XRectangle rect){ + // Match function to check if a rectangle already exists in the lookup table + bool match(XRectangle rect) + { bool isMatched = false; int minFreq = INT_MAX; this->min_freq_index = -1; + for (int i = 0; i < MAX_LOOKUP_COUNTS; i++) { - if(LOOKUPS[i].frequency == -1) break; + if (LOOKUPS[i].frequency == -1) + break; // Stop if we've reached unused entries + + // Increment the encounter count LOOKUPS[i].encounter++; - if (rect.x == LOOKUPS[i].rect.x && rect.y == LOOKUPS[i].rect.y && rect.width == LOOKUPS[i].rect.width && rect.height == LOOKUPS[i].rect.height) + + // Check if this rectangle matches + if (rect.x == LOOKUPS[i].rect.x && rect.y == LOOKUPS[i].rect.y && + rect.width == LOOKUPS[i].rect.width && rect.height == LOOKUPS[i].rect.height) { isMatched = true; this->lastMatchedFrequency = LOOKUPS[i].frequency; - LOOKUPS[i].frequency++; + LOOKUPS[i].frequency++; // Increment the frequency as it's matched + LOOKUPS[i].encounter = 0; // Reset encounter count as it's a match } - if(LOOKUPS[i].frequency < minFreq && LOOKUPS[i].encounter > 0){ + + // Track the least frequent, but encountered rectangle + if (LOOKUPS[i].frequency < minFreq && LOOKUPS[i].encounter > 0) + { minFreq = LOOKUPS[i].frequency; this->min_freq_index = i; } @@ -58,42 +72,56 @@ struct LOOKUP_MANAGER{ return isMatched; } - void push(XRectangle rect){ + // Push a new rectangle into the lookup manager or replace an existing one + void push(XRectangle rect) + { RectLookup lookup; lookup.rect = rect; - lookup.frequency = 0; - if(this->index < MAX_LOOKUP_COUNTS) { + lookup.frequency = 0; // New rectangle starts with a frequency of 0 + lookup.encounter = 0; // New rectangle hasn't been encountered yet + + // If there's room in the lookup array, add the new rectangle + if (this->index < MAX_LOOKUP_COUNTS) + { LOOKUPS[this->index++] = lookup; - } else { - if(this->min_freq_index >= 0 && this->min_freq_indexmin_freq_index] = lookup; } + } + else + { + // Replace the least frequently encountered rectangle if the table is full + if (this->min_freq_index >= 0 && this->min_freq_index < MAX_LOOKUP_COUNTS) + { + LOOKUPS[this->min_freq_index] = lookup; + } } } } myLOOKUP_MANAGER; -class OptimizationManager{ +class OptimizationManager +{ public: int checkOptimization(XRectangle rect); } optimizationManager; +// Function to check the optimization percentage based on frequency of the rectangle int OptimizationManager::checkOptimization(XRectangle rect) { if (myLOOKUP_MANAGER.match(rect)) { + // Based on frequency, return a reduced optimization level if (myLOOKUP_MANAGER.lastMatchedFrequency < 10) - return 80; + return 80; // High optimization if matched but not frequent else if (myLOOKUP_MANAGER.lastMatchedFrequency < 30) - return 50; + return 50; // Moderate optimization else - return 20; + return 20; // Low optimization for frequently matched rectangles } else { + // If not matched, add the rectangle and apply full optimization myLOOKUP_MANAGER.push(rect); return 100; } - return 100; + return 100; // Default case, should never be reached } -#endif \ No newline at end of file +#endif