datetime: make timezones not affect timestamps

This patch adds a fix making `datetime` handle `tz` or `tzoffset`
parameters properly.

Previously, changing a timezone altered the timestamp value but
preserved the time of day. Now you may enable new behavior making
specifying a timezone preserve the timestamp and update the represented
time of day.

This behavior could be achieved by setting a new compat option
`datetime_apply_timezone_action` to `'new'`. The option is going to be
enabled by default in 4.x.

Enabling new behavior involves the following.
1. Creating/setting a `timestamp` value with `tz`/`tzoffset` results
   with a day time corresponding to the timestamp value plus the
   timezone offset. E.g if we create a datetime object with a timestamp
   corresponding to 9:12 o'clock in `+0000` timezone and set
   `tzoffset = -60` the resulting timestamp will represent 8:12 o'clock.
2. Setting a new `tz`/`tzoffset` affects the time of day represented by
   the datetime. E.g. if you update a datetime object of 11:12 o'clock
   having `tzoffset = 180` with `tzoffset = 120` the resulting datetime
   object will represent 10:12.

Closes tarantool#10363

NO_DOC=will be in tarantool/doc#4720

Author: georgiy-belyanin

changelogs/unreleased/gh-10363-datetime-handle-timezone.md

@@ -0,0 +1,7 @@
+## bugfix/datetime
+
+* Proper behavior of applying timezones to datetime objects can
+  now be enabled by setting configuration option
+  `compat.datetime_apply_timezone_action` to `'new'`. It makes
+  changing timezones of `datetime` objects preserve the represented
+  timestamp and affect the represented time of day (gh-10363).

src/box/lua/config/descriptions.lua

@@ -338,6 +338,15 @@ I['compat.console_session_scope_vars'] = format_text([[
       from the console are written to globals
 ]])
 
+I['compat.datetime_apply_timezone_action'] = format_text([[
+    Controls the behavior of applying timezone to datetime objects:
+
+    - `new` (4.x default): applying a timezone doesn't affect the timestamp but
+      changes the represented time of day
+    - `old` (3.x default): applying a timezone alters the represented timestamp
+      but preserves the time of day
+]])
+
 I['compat.fiber_channel_close_mode'] = format_text([[
     Define the behavior of fiber channels after closing:
 

src/box/lua/config/instance_config.lua

@@ -2156,6 +2156,12 @@ return schema.new('instance_config', schema.record({
         }, {
             default = 'old',
         }),
+        datetime_apply_timezone_action = schema.enum({
+            'old',
+            'new',
+        }, {
+            default = 'old',
+        }),
         box_consider_system_spaces_synchronous = schema.enum({
             'old',
             'new',

src/lua/datetime.lua

@@ -1,7 +1,42 @@
+local compat = require('compat')
 local ffi = require('ffi')
 local buffer = require('buffer')
 local tz = require('timezones')
 
+-- This variable controls the behavior of applying timezone to
+-- datetime objects.
+--
+-- If set, setting a timezone for a datetime object doesn't
+-- affect the timestamp value but changes the represented time of
+-- day.
+-- Otherwise, the old behavior is used. Setting a timezone changes
+-- the timestamp value in that way to preserve the represented
+-- time of day.
+--
+-- This behavior will become default in 4.x.
+local applying_tz_preserves_timestamp
+
+local DATETIME_APPLY_TIMEZONE_ACTION_BRIEF = [[
+Whether applying timezone alters the represented time of day or the timestamp.
+The new behavior makes setting a timezone for a datetime object change the time
+of day and preserve the timestamp. The old behavior makes providing a timezone
+for a datetime object affect the timestamp but preserve the represented time of
+day.
+
+https://tarantool.io/compat/datetime_apply_timezone_action
+]]
+
+compat.add_option({
+    name = 'datetime_apply_timezone_action',
+    default = 'old',
+    obsolete = nil,
+    brief = DATETIME_APPLY_TIMEZONE_ACTION_BRIEF,
+    action = function(is_new)
+        applying_tz_preserves_timestamp = is_new
+    end,
+    run_action_now = true,
+})
+
 --[[
     `c-dt` library functions handles properly both positive and negative `dt`
     values, where `dt` is a number of dates since Rata Die date (0001-01-01).
@@ -621,6 +656,11 @@ local function datetime_new(obj)
             s = s - 1
             fraction = fraction + 1
         end
+
+        if applying_tz_preserves_timestamp then
+            s = s + (offset or 0) * 60
+        end
+
         -- if there are separate nsec, usec, or msec provided then
         -- timestamp should be integer
         if count_usec == 0 then
@@ -940,6 +980,13 @@ local function datetime_parse_from(str, obj)
     -- string.
     if date.tz == '' and date.tzoffset == 0 then
         datetime_set(date, { tzoffset = tzoffset, tz = tzname })
+
+        -- If working in a "preserve timestamp" mode the set
+        -- call changes the represented time of day and it should
+        -- be adjusted (gh-10363).
+        if applying_tz_preserves_timestamp then
+            time_localize(date, date.tzoffset)
+        end
     end
 
     return date, len
@@ -1152,13 +1199,27 @@ function datetime_set(self, obj)
         if tzname ~= nil then
             offset, self.tzindex = parse_tzname(sec_int, tzname)
         end
-        self.epoch = utc_secs(sec_int, offset)
+        if applying_tz_preserves_timestamp then
+            self.epoch = sec_int
+        else
+            self.epoch = utc_secs(sec_int, offset)
+        end
         self.nsec = nsec
         self.tzoffset = offset
 
         return self
     end
 
+    -- Only timezone is changed.
+    if not hms and not ymd and applying_tz_preserves_timestamp then
+        if tzname ~= nil then
+            offset, self.tzindex = parse_tzname(self.epoch, tzname)
+        end
+
+        self.tzoffset = offset
+        return self
+    end
+
     -- normalize time to UTC from current timezone
     time_delocalize(self)
 

test/app-luatest/datetime_test.lua

@@ -1,4 +1,5 @@
 local t = require('luatest')
+local compat = require('compat')
 local dt = require('datetime')
 
 local SUPPORTED_DATETIME_FORMATS = {
@@ -2098,3 +2099,151 @@ for supported_by, standard_cases in pairs(UNSUPPORTED_DATETIME_FORMATS) do
         end
     end
 end
+
+local TIMEZONES = {
+    {
+        tzname = 'Europe/Moscow',
+        tzoffset = 180,
+    },
+    {
+        tzname = 'Africa/Abidjan',
+        tzoffset = 0,
+    },
+    {
+        tzname = 'America/Argentina/Buenos_Aires',
+        tzoffset = -180,
+    },
+    {
+        tzname = 'Asia/Krasnoyarsk',
+        tzoffset = 420,
+    },
+    {
+        tzname = 'Pacific/Fiji',
+        tzoffset = 720,
+    }
+}
+
+local function tz_behavior_test_case(behavior, create_dt, expected)
+    return function()
+        t.assert_equals(compat.datetime_apply_timezone_action.default, 'old')
+        compat.datetime_apply_timezone_action = behavior
+
+        local dt_obj = create_dt()
+
+        t.assert_equals(dt_obj.hour, expected.hour)
+        t.assert_equals(dt_obj.timestamp, expected.timestamp)
+    end
+end
+
+-- Test new/old behavior of applying timezone to datetime objects
+-- (gh-10363).
+for _, case in ipairs(TIMEZONES) do
+    local function add_test_case(method, behavior, ...)
+        local testcase_name = ('test_tz_behavior_%s_%s_%s')
+            :format(method, case.tzname:gsub('/', '_'), behavior)
+        pg[testcase_name] = tz_behavior_test_case(behavior, ...)
+    end
+
+    local now = dt.now()
+    local create_dt = function()
+        return dt.new{timestamp = now.timestamp, tzoffset = case.tzoffset}
+    end
+
+    -- Note the following when using old behavior.
+    --
+    -- `datetime.new{timestamp = now.timestamp,
+    --               tzoffset = now.tzoffset}`
+    -- creates a datetime object representing time of day in GMT+0
+    -- corresponding to `now` time of day within the provided
+    -- timezone instead of creating a complete `now` copy.
+    add_test_case('create_tzoffset', 'old', create_dt,
+                  {hour = now.hour - now.tzoffset / 60,
+                   timestamp = now.timestamp - case.tzoffset * 60})
+    add_test_case('create_tzoffset', 'new', create_dt,
+                  {hour = (now.hour + (case.tzoffset - now.tzoffset) / 60) % 24,
+                   timestamp = now.timestamp})
+
+    create_dt = function()
+        return dt.new{timestamp = now.timestamp, tz = case.tzname}
+    end
+
+    add_test_case('create_tzname', 'old', create_dt,
+                  {hour = now.hour - now.tzoffset / 60,
+                   timestamp = now.timestamp - case.tzoffset * 60})
+    add_test_case('create_tzname', 'new', create_dt,
+                  {hour = (now.hour + (case.tzoffset - now.tzoffset) / 60) % 24,
+                   timestamp = now.timestamp})
+
+    create_dt = function()
+        return dt.new{timestamp = now.timestamp}:set{tzoffset = case.tzoffset}
+    end
+
+    add_test_case('set_tzoffset', 'old', create_dt,
+                  {hour = now.hour - now.tzoffset / 60,
+                   timestamp = now.timestamp - case.tzoffset * 60})
+    add_test_case('set_tzoffset', 'new', create_dt,
+                  {hour = (now.hour + (case.tzoffset - now.tzoffset) / 60) % 24,
+                   timestamp = now.timestamp})
+
+    create_dt = function()
+        return dt.new{timestamp = now.timestamp}:set{tz = case.tzname}
+    end
+
+    add_test_case('set_tzname', 'old', create_dt,
+                  {hour = now.hour - now.tzoffset / 60,
+                   timestamp = now.timestamp - case.tzoffset * 60})
+    add_test_case('set_tzname', 'new', create_dt,
+                  {hour = (now.hour + (case.tzoffset - now.tzoffset) / 60) % 24,
+                   timestamp = now.timestamp})
+end
+
+-- Test new/old behavior of applying timezone when parsing
+-- datetime objects.
+-- Adapted from app-tap/datetime.test.lua (gh-10363).
+pg.test_new_tz_behavior_parse = function()
+    t.assert_equals(compat.datetime_apply_timezone_action.default, 'old')
+    compat.datetime_apply_timezone_action = 'new'
+
+    t.assert(dt.parse("1970-01-01T01:00:00Z") ==
+            dt.new{year=1970, mon=1, day=1, hour=1, min=0, sec=0})
+    t.assert(dt.parse("1970-01-01T01:00:00Z", {format = 'iso8601'}) ==
+            dt.new{year=1970, mon=1, day=1, hour=1, min=0, sec=0})
+    t.assert(dt.parse("1970-01-01T01:00:00Z", {format = 'rfc3339'}) ==
+            dt.new{year=1970, mon=1, day=1, hour=1, min=0, sec=0})
+    t.assert(dt.parse("2020-01-01T01:00:00+00:00", {format = 'rfc3339'}) ==
+            dt.parse("2020-01-01T01:00:00+00:00", {format = 'iso8601'}))
+    t.assert(dt.parse("1970-01-01T02:00:00+02:00") ==
+            dt.new{year=1970, mon=1, day=1, hour=2, min=0, sec=0, tzoffset=120})
+
+    t.assert(dt.parse("1970-01-01T01:00:00", {tzoffset = 120}) ==
+        dt.new{year=1970, mon=1, day=1, hour=1, min=0, sec=0, tzoffset=120})
+    t.assert(dt.parse("1970-01-01T01:00:00", {tzoffset = '+0200'}) ==
+            dt.new{year=1970, mon=1, day=1, hour=1, min=0, sec=0, tzoffset=120})
+    t.assert(dt.parse("1970-01-01T01:00:00", {tzoffset = '+02:00'}) ==
+            dt.new{year=1970, mon=1, day=1, hour=1, min=0, sec=0, tzoffset=120})
+    t.assert(dt.parse("1970-01-01T01:00:00Z", {tzoffset = '+02:00'}) ==
+            dt.new{year=1970, mon=1, day=1, hour=1, min=0, sec=0, tzoffset=0})
+    t.assert(dt.parse("1970-01-01T01:00:00+01:00", {tzoffset = '+02:00'}) ==
+            dt.new{year=1970, mon=1, day=1, hour=1, min=0, sec=0, tzoffset=60})
+    t.assert(dt.parse('1998-11-25', { format = '%Y-%m-%d', tzoffset = 180 }) ==
+            dt.new({ year = 1998, month = 11, day = 25, tzoffset = 180 }))
+    t.assert(dt.parse('1998', { format = '%Y', tzoffset = '+03:00' }) ==
+            dt.new({ year = 1998, tzoffset = 180 }))
+
+    -- Testcases with override timezone by setting tz.
+    -- Timezone is not specified in a parsed string.
+    t.assert(dt.parse("1970-01-01T01:00:00", {tz = "MSK"}) ==
+        dt.new{year = 1970, mon = 1, day = 1,
+               hour = 1, min = 0, sec = 0, tzoffset=180})
+    -- Timezone is specified in a parsed string as a military timezone.
+    t.assert(dt.parse("1970-01-01T01:00:00Z", {tz = "Europe/Moscow"}) ==
+        dt.new{year = 1970, mon = 1, day = 1,
+               hour = 1, min = 0, sec = 0, tzoffset = 0})
+    -- Timezone is specified in a parsed string as an offset.
+    t.assert(dt.parse("1970-01-01T01:00:00+01:00", {tz = "Asia/Omsk"}) ==
+        dt.new{year = 1970, mon = 1, day = 1,
+               hour = 1, min = 0, sec = 0, tzoffset = 60})
+    -- Timezone is not specified in a parsed string and format is passed.
+    t.assert(dt.parse("1998-11-25", { format = "%Y-%m-%d", tz = "MSK" }) ==
+        dt.new{year = 1998, month = 11, day = 25, tzoffset = 180})
+end

test/config-luatest/cluster_config_schema_test.lua

@@ -439,6 +439,7 @@ g.test_defaults = function()
             box_error_unpack_type_and_code = 'old',
             console_session_scope_vars = 'old',
             wal_cleanup_delay_deprecation = 'old',
+            datetime_apply_timezone_action = 'old',
         },
         isolated = false,
     }