diff --git a/ios/accessibility/accessibility_control.go b/ios/accessibility/accessibility_control.go index 4ba99fbc..feaac4bf 100644 --- a/ios/accessibility/accessibility_control.go +++ b/ios/accessibility/accessibility_control.go @@ -1,6 +1,8 @@ package accessibility import ( + "context" + "encoding/base64" "fmt" dtx "github.com/danielpaulus/go-ios/ios/dtx_codec" @@ -14,6 +16,21 @@ type ControlInterface struct { channel *dtx.Channel } +// Direction represents navigation direction values used by AX service +type MoveDirection int32 + +const ( + DirectionPrevious MoveDirection = 3 + DirectionNext MoveDirection = 4 + DirectionFirst MoveDirection = 5 + DirectionLast MoveDirection = 6 +) + +// AXElementData represents the data returned from Move operations +type AXElementData struct { + PlatformElementValue string `json:"platformElementValue"` // Base64-encoded platform element data +} + func (a ControlInterface) readhostAppStateChanged() { for { msg := a.channel.ReceiveMethodCall("hostAppStateChanged:") @@ -115,7 +132,10 @@ func (a ControlInterface) SwitchToDevice() { a.deviceInspectorShowIgnoredElements(false) a.deviceSetAuditTargetPid(0) a.deviceInspectorFocusOnElement() - a.awaitHostInspectorCurrentElementChanged() + _, err := a.awaitHostInspectorCurrentElementChanged(context.Background()) + if err != nil { + log.Warnf("await element change failed during SwitchToDevice: %v", err) + } a.deviceInspectorPreviewOnElement() a.deviceHighlightIssue() } @@ -125,20 +145,72 @@ func (a ControlInterface) TurnOff() { a.deviceInspectorSetMonitoredEventType(0) a.awaitHostInspectorMonitoredEventTypeChanged() a.deviceInspectorFocusOnElement() - a.awaitHostInspectorCurrentElementChanged() + _, err := a.awaitHostInspectorCurrentElementChanged(context.Background()) + if err != nil { + log.Warnf("await element change failed during TurnOff: %v", err) + } a.deviceInspectorPreviewOnElement() a.deviceHighlightIssue() a.deviceInspectorShowVisuals(false) } -// GetElement moves the green selection rectangle one element further -func (a ControlInterface) GetElement() { +// Move navigates focus using the given direction and returns selected element data. +func (a ControlInterface) Move(ctx context.Context, direction MoveDirection) (AXElementData, error) { log.Info("changing") - a.deviceInspectorMoveWithOptions() - // a.deviceInspectorMoveWithOptions() + a.deviceInspectorMoveWithOptions(direction) + log.Info("before changed") - resp := a.awaitHostInspectorCurrentElementChanged() - log.Info("item changed", resp) + resp, err := a.awaitHostInspectorCurrentElementChanged(ctx) + if err != nil { + return AXElementData{}, err + } + + // Extraction path for platform element bytes: + // Value -> Value -> ElementValue_v1 -> Value -> Value -> PlatformElementValue_v1 -> Value ([]byte) + value, ok := resp["Value"].(map[string]interface{}) + if !ok { + return AXElementData{}, fmt.Errorf("resp[\"Value\"] is not a map, got %T", resp["Value"]) + } + + innerValue, ok := value["Value"].(map[string]interface{}) + if !ok { + return AXElementData{}, fmt.Errorf("Value[\"Value\"] is not a map, got %T", value["Value"]) + } + + elementValue, ok := innerValue["ElementValue_v1"].(map[string]interface{}) + if !ok { + return AXElementData{}, fmt.Errorf("ElementValue_v1 is not a map, got %T", innerValue["ElementValue_v1"]) + } + + axElement, ok := elementValue["Value"].(map[string]interface{}) + if !ok { + return AXElementData{}, fmt.Errorf("ElementValue_v1[\"Value\"] is not a map, got %T", elementValue["Value"]) + } + + valMap, ok := axElement["Value"].(map[string]interface{}) + if !ok { + return AXElementData{}, fmt.Errorf("AX element inner \"Value\" is not a map, got %T", axElement["Value"]) + } + + platformElement, ok := valMap["PlatformElementValue_v1"].(map[string]interface{}) + if !ok { + return AXElementData{}, fmt.Errorf("PlatformElementValue_v1 is not a map, got %T", valMap["PlatformElementValue_v1"]) + } + + byteArray, ok := platformElement["Value"].([]byte) + if !ok { + return AXElementData{}, fmt.Errorf("PlatformElementValue_v1[\"Value\"] is not a []byte, got %T", platformElement["Value"]) + } + encoded := base64.StdEncoding.EncodeToString(byteArray) + + return AXElementData{ + PlatformElementValue: encoded, + }, nil +} + +// GetElement moves the green selection rectangle one element further +func (a ControlInterface) GetElement(ctx context.Context) (AXElementData, error) { + return a.Move(ctx, DirectionNext) } func (a ControlInterface) UpdateAccessibilitySetting(name string, val interface{}) { @@ -159,14 +231,18 @@ func (a ControlInterface) ResetToDefaultAccessibilitySettings() error { return nil } -func (a ControlInterface) awaitHostInspectorCurrentElementChanged() map[string]interface{} { - msg := a.channel.ReceiveMethodCall("hostInspectorCurrentElementChanged:") +func (a ControlInterface) awaitHostInspectorCurrentElementChanged(ctx context.Context) (map[string]interface{}, error) { + msg, err := a.channel.ReceiveMethodCallWithTimeout(ctx, "hostInspectorCurrentElementChanged:") + if err != nil { + log.Errorf("Failed to receive hostInspectorCurrentElementChanged: %v", err) + return nil, fmt.Errorf("failed to receive hostInspectorCurrentElementChanged: %w", err) + } log.Info("received hostInspectorCurrentElementChanged") result, err := nskeyedarchiver.Unarchive(msg.Auxiliary.GetArguments()[0].([]byte)) if err != nil { panic(fmt.Sprintf("Failed unarchiving: %s this is a bug and should not happen", err)) } - return result[0].(map[string]interface{}) + return result[0].(map[string]interface{}), nil } func (a ControlInterface) awaitHostInspectorMonitoredEventTypeChanged() { @@ -175,13 +251,13 @@ func (a ControlInterface) awaitHostInspectorMonitoredEventTypeChanged() { log.Infof("hostInspectorMonitoredEventTypeChanged: was set to %d by the device", n[0]) } -func (a ControlInterface) deviceInspectorMoveWithOptions() { +func (a ControlInterface) deviceInspectorMoveWithOptions(direction MoveDirection) { method := "deviceInspectorMoveWithOptions:" options := nskeyedarchiver.NewNSMutableDictionary(map[string]interface{}{ "ObjectType": "passthrough", "Value": nskeyedarchiver.NewNSMutableDictionary(map[string]interface{}{ "allowNonAX": nskeyedarchiver.NewNSMutableDictionary(map[string]interface{}{"ObjectType": "passthrough", "Value": false}), - "direction": nskeyedarchiver.NewNSMutableDictionary(map[string]interface{}{"ObjectType": "passthrough", "Value": int32(4)}), + "direction": nskeyedarchiver.NewNSMutableDictionary(map[string]interface{}{"ObjectType": "passthrough", "Value": int32(direction)}), "includeContainers": nskeyedarchiver.NewNSMutableDictionary(map[string]interface{}{"ObjectType": "passthrough", "Value": true}), }), }) diff --git a/ios/accessibility/accessibility_integration_test.go b/ios/accessibility/accessibility_integration_test.go index 4ee74782..863e2b5c 100644 --- a/ios/accessibility/accessibility_integration_test.go +++ b/ios/accessibility/accessibility_integration_test.go @@ -10,7 +10,7 @@ import ( "github.com/danielpaulus/go-ios/ios/accessibility" ) -func TestIT(t *testing.T) { +func TestMove(t *testing.T) { device, err := ios.GetDevice("") if err != nil { t.Fatal(err) @@ -20,15 +20,31 @@ func TestIT(t *testing.T) { if err != nil { t.Fatal(err) } + defer conn.TurnOff() conn.SwitchToDevice() if err != nil { t.Fatal(err) } conn.EnableSelectionMode() - conn.GetElement() - conn.GetElement() - conn.TurnOff() - // conn.EnableSelectionMode() + t.Run("Test Move directions", func(t *testing.T) { + directions := []accessibility.MoveDirection{ + // newer ios(18+) devices sometimes doesn't bring focus to first element, so we need to move twice + accessibility.DirectionNext, + accessibility.DirectionNext, + accessibility.DirectionPrevious, + } + + for _, direction := range directions { + t.Logf("Testing direction: %v", direction) + element, err := conn.Move(direction) + if err != nil { + t.Logf("Move %v failed (expected on some devices): %v", direction, err) + continue + } + + t.Logf("Move %v succeeded: %+v", direction, element) + } + }) } diff --git a/ios/dtx_codec/channel.go b/ios/dtx_codec/channel.go index bba99765..6fac46d9 100644 --- a/ios/dtx_codec/channel.go +++ b/ios/dtx_codec/channel.go @@ -1,6 +1,7 @@ package dtx import ( + "context" "fmt" "sync" "time" @@ -46,6 +47,19 @@ func (d *Channel) ReceiveMethodCall(selector string) Message { return <-channel } +func (d *Channel) ReceiveMethodCallWithTimeout(ctx context.Context, selector string) (Message, error) { + d.mutex.Lock() + channel := d.registeredMethods[selector] + d.mutex.Unlock() + select { + case msg := <-channel: + return msg, nil + // context is cancelled because the timeout is exceeded + case <-ctx.Done(): + return Message{}, ctx.Err() + } +} + // MethodCall is the standard DTX style remote method invocation pattern. The ObjectiveC Selector goes as a NSKeyedArchiver.archived NSString into the // DTXMessage payload, and the arguments are separately NSKeyArchiver.archived and put into the Auxiliary DTXPrimitiveDictionary. It returns the response message and an error. func (d *Channel) MethodCall(selector string, args ...interface{}) (Message, error) { diff --git a/main.go b/main.go index ea240dc6..9926fc62 100644 --- a/main.go +++ b/main.go @@ -1798,7 +1798,7 @@ func startAx(device ios.DeviceEntry, arguments docopt.Opts) { } for i := 0; i < 3; i++ { - conn.GetElement() + conn.GetElement(context.Background()) time.Sleep(time.Second) } /* conn.GetElement()