let windowed tokens exchange information across heads a la talking heads prior to pointwise attention in sep-vit

lucidrains · lucidrains · commit d93cd84ccdb3 · 2022-03-31T15:22:24.000-07:00
diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 setup(
   name = 'vit-pytorch',
   packages = find_packages(exclude=['examples']),
-  version = '0.32.1',
+  version = '0.32.2',
   license='MIT',
   description = 'Vision Transformer (ViT) - Pytorch',
   author = 'Phil Wang',
diff --git a/vit_pytorch/sep_vit.py b/vit_pytorch/sep_vit.py
@@ -103,7 +103,7 @@ def __init__(
             nn.LayerNorm(dim_head),
             nn.GELU(),
             Rearrange('b h n c -> b (h c) n'),
-            nn.Conv1d(inner_dim, inner_dim * 2, 1, groups = heads),
+            nn.Conv1d(inner_dim, inner_dim * 2, 1),
             Rearrange('b (h c) n -> b h n c', h = heads),
         )
 

Original file line number	Diff line number	Diff line change
`@@ -103,7 +103,7 @@ def __init__(`
`103`	`103`	`nn.LayerNorm(dim_head),`
`104`	`104`	`nn.GELU(),`
`105`	`105`	`Rearrange('b h n c -> b (h c) n'),`
`106`		`- nn.Conv1d(inner_dim, inner_dim * 2, 1, groups = heads),`
	`106`	`+ nn.Conv1d(inner_dim, inner_dim * 2, 1),`
`107`	`107`	`Rearrange('b (h c) n -> b h n c', h = heads),`
`108`	`108`	`)`
`109`	`109`