| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, |
| * software distributed under the License is distributed on an |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| * KIND, either express or implied. See the License for the |
| * specific language governing permissions and limitations |
| * under the License. |
| */ |
| |
| #include "display-plan.h" |
| #include "display-priv.h" |
| #include "guacamole/client.h" |
| #include "guacamole/display.h" |
| #include "guacamole/fifo.h" |
| #include "guacamole/layer.h" |
| #include "guacamole/proctitle.h" |
| #include "guacamole/protocol-types.h" |
| #include "guacamole/protocol.h" |
| #include "guacamole/rect.h" |
| #include "guacamole/rwlock.h" |
| #include "guacamole/socket.h" |
| #include "guacamole/timestamp.h" |
| |
| #include <inttypes.h> |
| #include <limits.h> |
| #include <cairo/cairo.h> |
| #include <pthread.h> |
| |
| /** |
| * Returns a new Cairo surface representing the contents of the given dirty |
| * rectangle from the given layer. The returned surface must eventually be |
| * freed with a call to cairo_surface_destroy(). The graphical contents will be |
| * referenced from the layer's last_frame buffer. If sending the contents of a |
| * pending frame, that pending frame must have been copied over to the |
| * last_frame buffer before calling this function. |
| * |
| * @param display_layer |
| * The layer whose data should be referenced by the returned Cairo surface. |
| * |
| * @param dirty |
| * The region of the layer that should be referenced by the returned Cairo |
| * surface. |
| * |
| * @return |
| * A new Cairo surface that points to the given rectangle of image data |
| * from the last_frame buffer of the given layer. This surface must |
| * eventually be freed with a call to cairo_surface_destroy(). |
| */ |
| static cairo_surface_t* LFR_guac_display_layer_cairo_rect(guac_display_layer* display_layer, |
| guac_rect* dirty) { |
| |
| /* Get Cairo surface covering dirty rect */ |
| unsigned char* buffer = GUAC_DISPLAY_LAYER_STATE_MUTABLE_BUFFER(display_layer->last_frame, *dirty); |
| cairo_surface_t* rect; |
| |
| /* Use RGB24 if the image is fully opaque */ |
| if (display_layer->opaque) |
| rect = cairo_image_surface_create_for_data(buffer, |
| CAIRO_FORMAT_RGB24, guac_rect_width(dirty), |
| guac_rect_height(dirty), display_layer->last_frame.buffer_stride); |
| |
| /* Otherwise ARGB32 is needed, and the destination must be cleared */ |
| else |
| rect = cairo_image_surface_create_for_data(buffer, |
| CAIRO_FORMAT_ARGB32, guac_rect_width(dirty), |
| guac_rect_height(dirty), display_layer->last_frame.buffer_stride); |
| |
| return rect; |
| |
| } |
| |
| /** |
| * Sends instructions over the Guacamole connection to clear the given |
| * rectangle of the given layer if that layer is non-opaque. This is necessary |
| * prior to sending image data to layers with alpha transparency, as image data |
| * from multiple updates will otherwise be composited together. |
| * |
| * @param display_layer |
| * The layer that should possibly be cleared in preparation for a future |
| * drawing operation. |
| * |
| * @param dirty |
| * The rectangular region of the drawing operation. |
| */ |
| static void guac_display_layer_clear_non_opaque(guac_display_layer* display_layer, |
| guac_rect* dirty) { |
| |
| guac_display* display = display_layer->display; |
| const guac_layer* layer = display_layer->layer; |
| |
| guac_client* client = display->client; |
| guac_socket* socket = client->socket; |
| |
| /* Clear destination region only if necessary due to the relevant layer |
| * being non-opaque. The rect+cfill pair must be sent atomically to |
| * prevent interleaving with concurrent workers operating on the same |
| * layer. */ |
| if (!display_layer->opaque) { |
| |
| pthread_mutex_lock(&display_layer->path_lock); |
| |
| guac_protocol_send_rect(socket, layer, dirty->left, dirty->top, |
| guac_rect_width(dirty), guac_rect_height(dirty)); |
| |
| guac_protocol_send_cfill(socket, GUAC_COMP_ROUT, layer, |
| 0x00, 0x00, 0x00, 0xFF); |
| |
| pthread_mutex_unlock(&display_layer->path_lock); |
| |
| } |
| |
| } |
| |
| /** |
| * Returns an appropriate quality between 0 and 100 for lossy encoding |
| * depending on the current processing lag calculated for the given client. |
| * |
| * @param client |
| * The client for which the lossy quality is being calculated. |
| * |
| * @return |
| * A value between 0 and 100 inclusive which seems appropriate for the |
| * client based on lag measurements. |
| */ |
| static int guac_display_suggest_quality(guac_client* client) { |
| |
| int lag = guac_client_get_processing_lag(client); |
| |
| /* Scale quality linearly from 90 to 30 as lag varies from 20ms to 80ms */ |
| int quality = 90 - (lag - 20); |
| |
| /* Do not exceed 90 for quality */ |
| if (quality > 90) |
| return 90; |
| |
| /* Do not go below 30 for quality */ |
| if (quality < 30) |
| return 30; |
| |
| return quality; |
| |
| } |
| |
| /** |
| * Guesses whether a rectangle within a particular layer would be better |
| * compressed as PNG or using a lossy format like JPEG. Positive values |
| * indicate PNG is likely to be superior, while negative values indicate the |
| * opposite. |
| * |
| * @param layer |
| * The layer containing the image data to check. |
| * |
| * @param rect |
| * The rect to check within the given layer. |
| * |
| * @return |
| * Positive values if PNG compression is likely to perform better than |
| * lossy alternatives, or negative values if PNG is likely to perform |
| * worse. |
| */ |
| static int LFR_guac_display_layer_png_optimality(guac_display_layer* layer, |
| const guac_rect* rect) { |
| |
| int x, y; |
| |
| int num_same = 0; |
| int num_different = 1; |
| |
| /* Get buffer from layer */ |
| size_t stride = layer->last_frame.buffer_stride; |
| const unsigned char* buffer = GUAC_DISPLAY_LAYER_STATE_CONST_BUFFER(layer->last_frame, *rect); |
| |
| /* Image must be at least 1x1 */ |
| if (rect->right - rect->left < 1 || rect->bottom - rect->top< 1) |
| return 0; |
| |
| /* For each row */ |
| for (y = rect->top; y < rect->bottom; y++) { |
| |
| uint32_t* row = (uint32_t*) buffer; |
| uint32_t last_pixel = *(row++) | 0xFF000000; |
| |
| /* For each pixel in current row */ |
| for (x = rect->left + 1; x < rect->right; x++) { |
| |
| /* Get next pixel */ |
| uint32_t current_pixel = *(row++) | 0xFF000000; |
| |
| /* Update same/different counts according to pixel value */ |
| if (current_pixel == last_pixel) |
| num_same++; |
| else |
| num_different++; |
| |
| last_pixel = current_pixel; |
| |
| } |
| |
| /* Advance to next row */ |
| buffer += stride; |
| |
| } |
| |
| /* Return rough approximation of optimality for PNG compression. As PNG |
| * leverages lossless DEFLATE compression (which works by reducing the |
| * number of bytes required to represent repeated data), an approximation |
| * of the amount of repeated image data within the image is a reasonable |
| * approximation for how well an image will compress. */ |
| return 0x100 * num_same / num_different - 0x400; |
| |
| } |
| |
| /** |
| * Returns whether the given rectangle would be optimally encoded as JPEG |
| * rather than PNG. |
| * |
| * @param layer |
| * The layer to be queried. |
| * |
| * @param rect |
| * The rectangle to check. |
| * |
| * @param framerate |
| * The rate that the region covered by the given rectangle has historically |
| * been being updated within the given layer, in frames per second. |
| * |
| * @return |
| * Non-zero if the rectangle would be optimally encoded as JPEG, zero |
| * otherwise. |
| */ |
| static int LFR_guac_display_layer_should_use_jpeg(guac_display_layer* layer, |
| const guac_rect* rect, int framerate) { |
| |
| /* Do not use JPEG if lossless quality is required */ |
| if (layer->last_frame.lossless) |
| return 0; |
| |
| int rect_width = rect->right - rect->left; |
| int rect_height = rect->bottom - rect->top; |
| int rect_size = rect_width * rect_height; |
| |
| /* JPEG is preferred if: |
| * - frame rate is high enough |
| * - image size is large enough |
| * - PNG is not more optimal based on image contents */ |
| return framerate >= GUAC_DISPLAY_JPEG_FRAMERATE |
| && rect_size > GUAC_DISPLAY_JPEG_MIN_BITMAP_SIZE |
| && LFR_guac_display_layer_png_optimality(layer, rect) < 0; |
| |
| } |
| |
| /** |
| * Returns whether the given rectangle would be optimally encoded as WebP |
| * rather than PNG. |
| * |
| * @param layer |
| * The layer to be queried. |
| * |
| * @param rect |
| * The rectangle to check. |
| * |
| * @param framerate |
| * The rate that the region covered by the given rectangle has historically |
| * been being updated within the given layer, in frames per second. |
| * |
| * @return |
| * Non-zero if the rectangle would be optimally encoded as WebP, zero |
| * otherwise. |
| */ |
| static int LFR_guac_display_layer_should_use_webp(guac_display_layer* layer, |
| const guac_rect* rect, int framerate) { |
| |
| /* Do not use WebP if not supported */ |
| if (!guac_client_supports_webp(layer->display->client)) |
| return 0; |
| |
| /* WebP is preferred if: |
| * - frame rate is high enough |
| * - PNG is not more optimal based on image contents */ |
| return framerate >= GUAC_DISPLAY_JPEG_FRAMERATE |
| && LFR_guac_display_layer_png_optimality(layer, rect) < 0; |
| |
| } |
| |
| void* guac_display_worker_thread(void* data) { |
| |
| /* Thread name display-wrk: one worker in the display pool; encodes and |
| * sends graphical updates for dirty layer regions. */ |
| guac_thread_name_set("display-wrk"); |
| |
| int framerate; |
| int has_outstanding_frames = 0; |
| |
| guac_display* display = (guac_display*) data; |
| guac_client* client = display->client; |
| guac_socket* socket = client->socket; |
| |
| guac_display_plan_operation op; |
| while (guac_fifo_dequeue_and_lock(&display->ops, &op)) { |
| |
| /* Notify any watchers of render_state that a frame is now in progress */ |
| guac_flag_set_and_lock(&display->render_state, GUAC_DISPLAY_RENDER_STATE_FRAME_IN_PROGRESS); |
| guac_flag_clear(&display->render_state, GUAC_DISPLAY_RENDER_STATE_FRAME_NOT_IN_PROGRESS); |
| guac_flag_unlock(&display->render_state); |
| |
| /* NOTE: Any thread that locks the operation queue can know that there |
| * are no pending operations in progress if the queue is empty and |
| * there are no active workers */ |
| display->active_workers++; |
| guac_fifo_unlock(&display->ops); |
| |
| guac_rwlock_acquire_read_lock(&display->last_frame.lock); |
| guac_display_layer* display_layer = op.layer; |
| switch (op.type) { |
| |
| case GUAC_DISPLAY_PLAN_OPERATION_IMG: |
| |
| framerate = INT_MAX; |
| if (op.current_frame > op.last_frame) |
| framerate = 1000 / (op.current_frame - op.last_frame); |
| |
| guac_rect* dirty = &op.dest; |
| |
| /* TODO: Determine whether to use PNG/WebP/JPEG purely |
| * based on whether lossless encoding is required, the |
| * expected time until another frame is received (time |
| * since last frame), and estimated encoding times. The |
| * time allowed per update should be divided up |
| * proportionately based on the dirty_size of the update. */ |
| |
| /* TODO: Stream PNG/WebP/JPEG using progressive encoding such |
| * that a frame that is currently being encoded can be |
| * preempted by the next frame, with the connected client then |
| * simply receiving a lower-quality intermediate frame. If |
| * necessary, progressive encoding can be achieved by manually |
| * dividing images into multiple reduced-resolution stages, |
| * such that each image streamed is actually only one quarter |
| * the size of the original image. Compositing via Guacamole |
| * protocol instructions can reassemble those stages. */ |
| |
| cairo_surface_t* rect = LFR_guac_display_layer_cairo_rect(display_layer, dirty); |
| const guac_layer* layer = display_layer->layer; |
| |
| /* Clear relevant rect of destination layer if necessary to |
| * ensure fresh data is not drawn on top of old data for layers |
| * with alpha transparency */ |
| guac_display_layer_clear_non_opaque(display_layer, dirty); |
| |
| /* Prefer WebP when reasonable */ |
| if (LFR_guac_display_layer_should_use_webp(display_layer, dirty, framerate)) |
| guac_client_stream_webp(client, socket, GUAC_COMP_OVER, layer, |
| dirty->left, dirty->top, rect, |
| guac_display_suggest_quality(client), |
| display_layer->last_frame.lossless ? 1 : 0); |
| |
| /* If not WebP, JPEG is the next best (lossy) choice */ |
| else if (display_layer->opaque && LFR_guac_display_layer_should_use_jpeg(display_layer, dirty, framerate)) |
| guac_client_stream_jpeg(client, socket, GUAC_COMP_OVER, layer, |
| dirty->left, dirty->top, rect, |
| guac_display_suggest_quality(client)); |
| |
| /* Use PNG if no lossy formats are appropriate */ |
| else |
| guac_client_stream_png(client, socket, GUAC_COMP_OVER, |
| layer, dirty->left, dirty->top, rect); |
| |
| cairo_surface_destroy(rect); |
| break; |
| |
| case GUAC_DISPLAY_PLAN_OPERATION_COPY: |
| case GUAC_DISPLAY_PLAN_OPERATION_RECT: |
| guac_client_log(client, GUAC_LOG_DEBUG, "Operation type %i " |
| "should NOT be present in the set of operations given " |
| "to guac_display worker thread. All operations except " |
| "IMG and NOP are handled during the initial, " |
| "single-threaded flush step. This is likely a bug.", |
| op.type); |
| break; |
| |
| case GUAC_DISPLAY_PLAN_OPERATION_NOP: |
| /* Do nothing */ |
| break; |
| |
| } |
| |
| guac_fifo_lock(&display->ops); |
| |
| /* If we're the only active worker and there are no further operations |
| * pending, we've reached the end of the frame, and this is the worker |
| * that will be sending that boundary to connected users */ |
| if (!(display->ops.state.value & GUAC_FIFO_STATE_NONEMPTY) && display->active_workers == 1) { |
| |
| /* Update the mouse cursor if it's been changed since the |
| * last frame */ |
| guac_display_layer* cursor = display->cursor_buffer; |
| if (!guac_rect_is_empty(&cursor->last_frame.dirty)) { |
| guac_protocol_send_cursor(client->socket, |
| display->last_frame.cursor_hotspot_x, |
| display->last_frame.cursor_hotspot_y, |
| cursor->layer, 0, 0, |
| cursor->last_frame.width, |
| cursor->last_frame.height); |
| } |
| |
| /* Allow connected clients to move forward with rendering */ |
| guac_client_end_multiple_frames(client, display->last_frame.frames); |
| |
| /* While connected clients moves forward with rendering, |
| * commit any changed contents to client-side backing buffer */ |
| guac_display_layer* current = display->last_frame.layers; |
| while (current != NULL) { |
| |
| /* Save a copy of the changed region if the layer has |
| * been modified since the last frame */ |
| guac_rect* dirty = ¤t->last_frame.dirty; |
| if (!guac_rect_is_empty(dirty)) { |
| |
| int x = dirty->left; |
| int y = dirty->top; |
| int width = guac_rect_width(dirty); |
| int height = guac_rect_height(dirty); |
| |
| /* Ensure destination region is cleared out first if the alpha channel need be considered, |
| * as GUAC_COMP_OVER is significantly faster than GUAC_COMP_SRC on the browser side */ |
| if (!current->opaque) { |
| guac_protocol_send_rect(client->socket, current->last_frame_buffer, x, y, width, height); |
| guac_protocol_send_cfill(client->socket, GUAC_COMP_RATOP, current->last_frame_buffer, |
| 0x00, 0x00, 0x00, 0x00); |
| } |
| |
| guac_protocol_send_copy(client->socket, |
| current->layer, x, y, width, height, |
| GUAC_COMP_OVER, current->last_frame_buffer, x, y); |
| |
| } |
| |
| current = current->last_frame.next; |
| |
| } |
| |
| /* This is now absolutely everything for the current frame, |
| * and it's safe to flush any outstanding data */ |
| guac_socket_flush(client->socket); |
| |
| /* Notify any watchers of render_state that a frame is no longer in progress */ |
| guac_flag_set_and_lock(&display->render_state, GUAC_DISPLAY_RENDER_STATE_FRAME_NOT_IN_PROGRESS); |
| guac_flag_clear(&display->render_state, GUAC_DISPLAY_RENDER_STATE_FRAME_IN_PROGRESS); |
| guac_flag_unlock(&display->render_state); |
| |
| has_outstanding_frames = display->frame_deferred; |
| |
| } |
| |
| display->active_workers--; |
| guac_fifo_unlock(&display->ops); |
| |
| guac_rwlock_release_lock(&display->last_frame.lock); |
| |
| /* Trigger additional flush if frames were completed while we were |
| * still processing the previous frame */ |
| if (has_outstanding_frames) { |
| guac_display_end_multiple_frames(display, 0); |
| has_outstanding_frames = 0; |
| } |
| |
| } |
| |
| return NULL; |
| |
| } |
