Fixed Host normalization for certain cases + support X-Forwarded-Host

CHANGELOG.md

@@ -440,4 +440,11 @@ Magento CE 1.8+ or EE 1.13+, see [these instructions](https://github.com/nexcess
   * [#1155] Update TECHNICAL_NOTES.md. (@GLips)
 
 ### RELEASE-0.6.9
+  * [#1162] Support PHP7. (@allardhoeve)
+  * [#1173] Fix load balancing for Varnish 4. (@kleinmann)
+  * [#1182] Update version-3.vcl normalisation. (@gewaechshaus)
+
+### RELEASE-0.7.0
+
+
 

app/code/community/Nexcessnet/Turpentine/Block/Poll/ActivePoll.php

@@ -21,11 +21,19 @@
 
 class Nexcessnet_Turpentine_Block_Poll_ActivePoll extends Mage_Poll_Block_ActivePoll {
 
-    public function setTemplate($template)
-    {
-        $this->_template = $template;
-        $this->setPollTemplate('turpentine/ajax.phtml', 'poll');
-        $this->setPollTemplate('turpentine/ajax.phtml', 'results');
-        return $this;
-    }
+	public function setTemplate($template)
+	{
+		if ((Mage::getConfig()->getModuleConfig('Mage_Poll')->is('active', 'true')) &&
+		(!Mage::getStoreConfig('advanced/modules_disable_output/Mage_Poll')))
+		{
+			$this->_template = $template;
+			$this->setPollTemplate('turpentine/ajax.phtml', 'poll');
+			$this->setPollTemplate('turpentine/ajax.phtml', 'results');
+		}
+		else
+		{
+			// Mage_Poll is disabled, so do nothing
+		}
+		return $this;
+	}
 }

app/code/community/Nexcessnet/Turpentine/Model/Config/Select/NormalizeHost.php

@@ -0,0 +1,34 @@
+<?php
+
+/** 
+ * Nexcess.net Turpentine Extension for Magento
+ * Copyright (C) 2012  Nexcess.net L.L.C.
+ * 
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */ 
+
+class Nexcessnet_Turpentine_Model_Config_Select_NormalizeHost {
+    /**
+     * @return array
+     */
+    public function toOptionArray() {
+        $helper = Mage::helper('turpentine');
+        return array(
+            array('value'=>'yes', 'label'=>Mage::helper('adminhtml')->__('Enable')),
+            array('value'=>'yes_forwarded_host', 'label'=>$helper->__('Yes, use HTTP X-Forwarded-Host Header')),
+            array('value'=>'no', 'label'=>Mage::helper('adminhtml')->__('Disable')),
+        );
+    }
+}

app/code/community/Nexcessnet/Turpentine/Model/Observer/Esi.php

@@ -226,12 +226,19 @@ public function injectEsi($eventObject) {
             $debugHelper->logInfo(
                 'Checking ESI block candidate: %s',
                 $blockObject->getNameInLayout() ? $blockObject->getNameInLayout() : $blockObject->getModuleName() );
-        }
+
+            $debugHelper->logInfo( "-- block testing: shouldResponseUseEsi = " . $esiHelper->shouldResponseUseEsi());
+            $debugHelper->logInfo( "-- block testing: instanceof Mage_Core_Block_Template = " . $blockObject instanceof Mage_Core_Block_Template );
+            $debugHelper->logInfo( "-- block testing: Esi Options = " . print_r($blockObject->getEsiOptions(), true) );
+        }        
         if ($esiHelper->shouldResponseUseEsi() &&
                 $blockObject instanceof Mage_Core_Block_Template &&
                 $esiOptions = $blockObject->getEsiOptions()) {
 
             if ((isset($esiOptions['disableEsiInjection'])) && ($esiOptions['disableEsiInjection'] == 1)) { 
+                if ($esiHelper->getEsiBlockLogEnabled()) {
+                    $debugHelper->logInfo("-- ESI Injection disabled");
+                }
                 return;
             }
 

app/code/community/Nexcessnet/Turpentine/Model/Varnish/Admin/Socket.php

@@ -499,6 +499,9 @@ protected function _command($verb, $okCode = 200) {
                 "Got unexpected response code from Varnish: %d\n%s",
                 $response['code'], $response['text'] ));
         } else {
+            if (Mage::getStoreConfig('turpentine_varnish/general/varnish_log_commands')) { 
+                Mage::helper('turpentine/debug')->logDebug('VARNISH command sent: ' . $data);
+            }
             return $response;
         }
     }

app/code/community/Nexcessnet/Turpentine/Model/Varnish/Configurator/Abstract.php

@@ -110,8 +110,8 @@ public function save($generatedConfig) {
      * @return string
      */
     protected function _getVclTemplateFilename($baseFilename) {
-        $extensionDir = Mage::getModuleDir('', 'Nexcessnet_Turpentine');
-        return sprintf('%s/misc/%s', $extensionDir, $baseFilename);
+           $extensionDir = Mage::getModuleDir('', 'Nexcessnet_Turpentine');
+           return sprintf('%s/misc/%s', $extensionDir, $baseFilename);
     }
 
     /**
@@ -136,6 +136,23 @@ protected function _getCustomIncludeFilename() {
             array('root_dir' => Mage::getBaseDir()) );
     }
 
+
+    /**
+     * Get the custom VCL template, if it exists
+     * Returns 'null' if the file doesn't exist
+     *
+     * @return string
+     */
+    protected function _getCustomTemplateFilename() {
+        $filePath = $this->_formatTemplate(
+            Mage::getStoreConfig('turpentine_varnish/servers/custom_vcl_template'),
+            array('root_dir' => Mage::getBaseDir())
+        );
+        if (is_file($filePath)) { return $filePath; }
+        else { return null; }
+    }
+
+
     /**
      * Format a template string, replacing {{keys}} with the appropriate values
      * and remove unspecified keys
@@ -173,7 +190,11 @@ protected function _vcl_call($subroutine) {
      */
     protected function _getAdminFrontname() {
         if (Mage::getStoreConfig('admin/url/use_custom_path')) {
-            return Mage::getStoreConfig('admin/url/custom_path');
+            if(Mage::getStoreConfig('web/url/use_store')) {
+                return Mage::getModel('core/store')->load(0)->getCode() . "/" . Mage::getStoreConfig('admin/url/custom_path');
+            } else {
+                return Mage::getStoreConfig('admin/url/custom_path');
+            }
         } else {
             return (string) Mage::getConfig()->getNode(
                 'admin/routers/adminhtml/args/frontName' );
@@ -769,16 +790,6 @@ protected function _vcl_sub_normalize_user_agent() {
         $tpl = <<<EOS
 if (req.http.User-Agent ~ "iP(?:hone|ad|od)|BlackBerry|Palm|Googlebot-Mobile|Mobile|mobile|mobi|Windows Mobile|Safari Mobile|Android|Opera (?:Mini|Mobi)") {
         set req.http.X-Normalized-User-Agent = "mobile";
-    } else if (req.http.User-Agent ~ "MSIE") {
-        set req.http.X-Normalized-User-Agent = "msie";
-    } else if (req.http.User-Agent ~ "Firefox") {
-        set req.http.X-Normalized-User-Agent = "firefox";
-    } else if (req.http.User-Agent ~ "Chrome") {
-        set req.http.X-Normalized-User-Agent = "chrome";
-    } else if (req.http.User-Agent ~ "Safari") {
-        set req.http.X-Normalized-User-Agent = "safari";
-    } else if (req.http.User-Agent ~ "Opera") {
-        set req.http.X-Normalized-User-Agent = "opera";
     } else {
         set req.http.X-Normalized-User-Agent = "other";
     }
@@ -823,6 +834,21 @@ protected function _vcl_sub_normalize_host() {
             'normalize_host_target' => $this->_getNormalizeHostTarget() ));
     }
 
+    /**
+     * Get the Host normalization sub routine
+     *
+     * @return string
+     */
+    protected function _vcl_sub_normalize_host_forwarded() {
+        $tpl = <<<EOS
+if (req.http.X-Forwarded-Host)
+{
+    set req.http.Host = req.http.X-Forwarded-Host;
+}
+EOS;
+        return $this->_formatTemplate($tpl, array());
+    }
+
     /**
      * Get the hostname for cookie normalization
      *
@@ -990,8 +1016,13 @@ protected function _getTemplateVars() {
         if (Mage::getStoreConfig('turpentine_vcl/normalization/user_agent')) {
             $vars['normalize_user_agent'] = $this->_vcl_sub_normalize_user_agent();
         }
-        if (Mage::getStoreConfig('turpentine_vcl/normalization/host')) {
-            $vars['normalize_host'] = $this->_vcl_sub_normalize_host();
+        Mage::log(Mage::getStoreConfig('turpentine_vcl/normalization/host'));
+        if (Mage::getStoreConfig('turpentine_vcl/normalization/host') && Mage::getStoreConfig('turpentine_vcl/normalization/host') != 'no') {
+            if (Mage::getStoreConfig('turpentine_vcl/normalization/host') == "yes_forwarded_host") {
+                $vars['normalize_host'] = $this->_vcl_sub_normalize_host_forwarded();
+            } else {
+                $vars['normalize_host'] = $this->_vcl_sub_normalize_host();
+            }
         }
         if (Mage::getStoreConfig('turpentine_vcl/normalization/cookie_regex')) {
             $vars['normalize_cookie_regex'] = $this->_getNormalizeCookieRegex();

app/code/community/Nexcessnet/Turpentine/Model/Varnish/Configurator/Version3.php

@@ -31,7 +31,14 @@ class Nexcessnet_Turpentine_Model_Varnish_Configurator_Version3
      * @return string
      */
     public function generate($doClean = true) {
-        $tplFile = $this->_getVclTemplateFilename(self::VCL_TEMPLATE_FILE);
+        // first, check if a custom template is set
+        $customTemplate = $this->_getCustomTemplateFilename();
+        if ($customTemplate) { 
+            $tplFile = $customTemplate;
+        }
+        else { 
+            $tplFile = $this->_getVclTemplateFilename(self::VCL_TEMPLATE_FILE);
+        }
         $vcl = $this->_formatTemplate(file_get_contents($tplFile),
             $this->_getTemplateVars());
         return $doClean ? $this->_cleanVcl($vcl) : $vcl;

app/code/community/Nexcessnet/Turpentine/Model/Varnish/Configurator/Version4.php

@@ -31,7 +31,14 @@ class Nexcessnet_Turpentine_Model_Varnish_Configurator_Version4
      * @return string
      */
     public function generate($doClean = true) {
-        $tplFile = $this->_getVclTemplateFilename(self::VCL_TEMPLATE_FILE);
+        // first, check if a custom template is set
+        $customTemplate = $this->_getCustomTemplateFilename();
+        if ($customTemplate) { 
+            $tplFile = $customTemplate;
+        }
+        else { 
+            $tplFile = $this->_getVclTemplateFilename(self::VCL_TEMPLATE_FILE);
+        }
         $vcl = $this->_formatTemplate(file_get_contents($tplFile),
             $this->_getTemplateVars());
         return $doClean ? $this->_cleanVcl($vcl) : $vcl;
@@ -55,6 +62,134 @@ protected function _getTemplateVars() {
         $vars = parent::_getTemplateVars();
         $vars['advanced_session_validation'] =
             $this->_getAdvancedSessionValidation();
+
+        if (Mage::getStoreConfig('turpentine_vcl/backend/load_balancing') != 'no') {
+            $vars['directors']          = $this->_vcl_directors();
+            $vars['admin_backend_hint'] = 'vdir_admin.backend()';
+            $vars['set_backend_hint']   = 'set req.backend_hint = vdir.backend();';
+        } else {
+            $vars['directors']          = '';
+            $vars['admin_backend_hint'] = 'admin';
+            $vars['set_backend_hint']   = '';
+        }
+
         return $vars;
     }
+
+    protected function _vcl_directors()
+    {
+        $tpl = <<<EOS
+    new vdir       = directors.round_robin();
+    new vdir_admin = directors.round_robin();
+EOS;
+
+        if ('yes_admin' == Mage::getStoreConfig('turpentine_vcl/backend/load_balancing')) {
+            $adminBackendNodes = Mage::helper('turpentine/data')->cleanExplode(PHP_EOL,
+                Mage::getStoreConfig('turpentine_vcl/backend/backend_nodes_admin'));
+        } else {
+            $adminBackendNodes = Mage::helper('turpentine/data')->cleanExplode(PHP_EOL,
+                Mage::getStoreConfig('turpentine_vcl/backend/backend_nodes'));
+        }
+
+        $backendNodes = Mage::helper('turpentine/data')->cleanExplode(PHP_EOL,
+            Mage::getStoreConfig('turpentine_vcl/backend/backend_nodes'));
+
+        for($i = 0, $iMax = count($backendNodes); $i < $iMax; $i++) {
+            $tpl .= <<<EOS
+    vdir.add_backend(web{$i});
+EOS;
+        }
+
+        for($i = 0, $iMax = count($adminBackendNodes); $i < $iMax; $i++) {
+            $tpl .= <<<EOS
+    vdir_admin.add_backend(webadmin{$i});
+EOS;
+        }
+
+        $vars = array();
+
+        return $this->_formatTemplate($tpl, $vars);
+    }
+
+    /**
+     * Format a VCL director declaration, for load balancing
+     *
+     * @param string $name           name of the director, also used to select config settings
+     * @param array  $backendOptions options for each backend
+     * @return string
+     */
+    protected function _vcl_director($name, $backendOptions) {
+        $tpl = <<<EOS
+{{backends}}
+EOS;
+        if ('admin' == $name && 'yes_admin' == Mage::getStoreConfig('turpentine_vcl/backend/load_balancing')) {
+            $backendNodes = Mage::helper('turpentine/data')->cleanExplode(PHP_EOL,
+                Mage::getStoreConfig('turpentine_vcl/backend/backend_nodes_admin'));
+            $probeUrl = Mage::getStoreConfig('turpentine_vcl/backend/backend_probe_url_admin');
+            $prefix = 'admin';
+        } else {
+            $backendNodes = Mage::helper('turpentine/data')->cleanExplode(PHP_EOL,
+                Mage::getStoreConfig('turpentine_vcl/backend/backend_nodes'));
+            $probeUrl = Mage::getStoreConfig('turpentine_vcl/backend/backend_probe_url');
+
+            if('admin' == $name) {
+                $prefix = 'admin';
+            } else {
+                $prefix = '';
+            }
+        }
+
+        $backends = '';
+        $number = 0;
+        foreach ($backendNodes as $backendNode) {
+            $parts = explode(':', $backendNode, 2);
+            $host = (empty($parts[0])) ? '127.0.0.1' : $parts[0];
+            $port = (empty($parts[1])) ? '80' : $parts[1];
+            $backends .= $this->_vcl_director_backend($host, $port, $prefix . $number, $probeUrl, $backendOptions);
+
+            $number++;
+        }
+        $vars = array(
+            'name' => $name,
+            'backends' => $backends
+        );
+        return $this->_formatTemplate($tpl, $vars);
+    }
+
+    /**
+     * Format a VCL backend declaration to put inside director
+     *
+     * @param string $host       backend host
+     * @param string $port       backend port
+     * @param string $descriptor backend descriptor
+     * @param string $probeUrl   URL to check if backend is up
+     * @param array  $options    extra options for backend
+     * @return string
+     */
+    protected function _vcl_director_backend($host, $port, $descriptor, $probeUrl = '', $options = array()) {
+        $tpl = <<<EOS
+        backend web{$descriptor} {
+            .host = "{{host}}";
+            .port = "{{port}}";
+{{probe}}
+
+EOS;
+        $vars = array(
+            'host'  => $host,
+            'port'  => $port,
+            'probe' => ''
+        );
+        if ( ! empty($probeUrl)) {
+            $vars['probe'] = $this->_vcl_get_probe($probeUrl);
+        }
+        $str = $this->_formatTemplate($tpl, $vars);
+        foreach ($options as $key => $value) {
+            $str .= sprintf('            .%s = %s;', $key, $value).PHP_EOL;
+        }
+        $str .= <<<EOS
+        }
+
+EOS;
+        return $str;
+    }
 }

app/code/community/Nexcessnet/Turpentine/controllers/EsiController.php

@@ -253,6 +253,15 @@ protected function _getEsiBlock($esiData) {
                 $layout->generateBlocks($node);
             }
         }
+        if ($roots = $layout->getNode()->xpath('//block[@name=\'root\']')) {
+            foreach (array('formkey') as $globalBlock) {
+                if ($blocks = $layout->getNode()->xpath(sprintf('//block[@name=\'%s\']', $globalBlock))) {
+                    $dummy = $roots[0]->addChild('reference');
+                    $dummy->appendChild($blocks[0]);
+                    $layout->generateBlocks($dummy);
+                }
+            }
+        }
         $block = $layout->getBlock($esiData->getNameInLayout());
 
         if ( ! $this->getFlag('', self::FLAG_NO_DISPATCH_BLOCK_EVENT)) {

app/code/community/Nexcessnet/Turpentine/etc/config.xml

@@ -47,6 +47,7 @@
                 <server_list>127.0.0.1:6082</server_list>
                 <config_file>{{root_dir}}/var/default.vcl</config_file>
                 <custom_include_file>{{root_dir}}/app/code/community/Nexcessnet/Turpentine/misc/custom_include.vcl</custom_include_file>
+                <custom_vcl_template></custom_vcl_template>
             </servers>
         </turpentine_varnish>
         <turpentine_vcl>