Fix CLIP logit scale inplace clip behavior (#130)

* add swin transformer

* add swin transformer

* add swin transformer

* fix clip logit scale behavior

Author: firestonelib

passl/modeling/backbones/clip.py

@@ -314,7 +314,8 @@ def encode_text(self, text):
         return x
 
     def clip_logit_scale(self):
-        self.logit_scale.clip(-4.6, 4.6)
+        logit_scale_buffer = self.logit_scale.clip_(-4.6, 4.6)
+        logit_scale_buffer._share_buffer_to(self.logit_scale)
 
     def forward(self, image, text, is_train=True):
         image_features = self.encode_image(image)