Brainstorm's snippets (20/267)


  mirror_gitlab_projects
gitlab
  Github: Checking out pull requests locally
github
  django: how to filter out GET static and media messages with logging
django, logging
  elapsed_time():: Human readable time interval between two values of time.time()
python
  Symmetrical editing fo a M2M relation in Django admin
m2m
  Sort a list of dictionaries in Python3
sort
  Prevent Mac from falling asleep
mac, sleep
  How to Install and upgrade Node.js and NPM on Ubuntu
ubuntu, npm, nodejs
  cmp: Replacement for built-in function cmp that was removed in Python 3
python3
  Network Configuration in Debian 10
networking, debian
  Django & HTTPS tricks
https
  PostgreSQL ODBC connection from Ubuntu
odbc
  CookiesDirective for Privacy Policy
javascript
  Docker notes
docker
  django-imagekit: Generate predictable filenames in cache
imagekit
  Send an HTTP request using Telnet
telnet, http
  Ngrok references
ngrok
  Building Python 3 on embedded Linux
linux, python3
  django-parler: Make existing fields translatable
parler
  django-imagekit: How to cleanup cache backend and remove generated thumbnails from storage
thumbnail, django-imagekit

  mirror_gitlab_projects

#!/usr/bin/env python3
import gitlab
import os
import sys
import signal
import argparse
import logging
# requires: python-gitlab


GITLAB_URL = 'https://gitlab.somewhere.com'
PRIVATE_TOKEN = '********************'


# Get an instance of a logger
logger = logging.getLogger('main_module')
gl = gitlab.Gitlab(GITLAB_URL, private_token=PRIVATE_TOKEN)


def say_cwd():
    logger.debug('cwd: "%s"', os.getcwd())

def run_command(command):
    logger.info(command)
    rc = os.system(command)
    return rc

def new_dir(name):
    say_cwd()
    logger.info('mkdir: "%s"' % name)
    os.mkdir(name)

def clone_repo(url):
    say_cwd()
    run_command("git clone " + url)

def fetch_all():
    say_cwd()
    run_command("git fetch --all")


def mirror_project(group, project, wiki):
    logger.info('handle %s ...', 'wiki' if wiki else 'code')
    cwd = os.getcwd()
    say_cwd()

    try:
        # Move into group folder
        if not os.path.isdir(group.path):
            new_dir(group.path)
        os.chdir(group.path)

        # Select either code repo or wiki
        path = project.path
        url = project.ssh_url_to_repo
        if wiki:
            path += '.wiki'
            url = url[:-4] + '.wiki.git'

        # Clone repo, in case
        if not os.path.isdir(path):
            clone_repo(url)

        # Update repo
        os.chdir(path)
        fetch_all()

    finally:
        os.chdir(cwd)


def signal_handler(signal, frame):
    sys.exit(0)


def main():

    signal.signal(signal.SIGINT, signal_handler)

    parser = argparse.ArgumentParser(description='Clone and/or fetch repos and wikis from remote Gitlab host')
    parser.add_argument('-l', '--logfile', metavar='logfile', help='log filename; defaults to stdout')
    parser.add_argument('-v', '--verbosity', type=int, choices=range(4), default=2, action='store', help="log verbosity level")
    parser.add_argument('-g', '--group', help='filter by group path')
    args = parser.parse_args()

    # Setup logging
    loglevel = logging.WARN
    if args.verbosity == 0:
        loglevel = logging.ERROR
    elif args.verbosity == 1:  # default
        loglevel = logging.WARN
    elif args.verbosity == 2:
        loglevel = logging.INFO
    elif args.verbosity > 2:
        loglevel = logging.DEBUG

    logging.basicConfig(
        filename=args.logfile,
        level=loglevel,
        format='%(asctime)s|%(levelname)-8s| %(message)s',
    )

    filter_group = args.group

    path = os.path.realpath(__file__)
    os.chdir(os.path.split(path)[0])
    say_cwd()

    groups = gl.groups.list(per_page=1000)
    for g in groups:

        group = gl.groups.get(g.id)
        if filter_group is not None and filter_group != group.path:
            continue

        logger.info('GROUP:   "%s"', group.path)

        projects = group.projects.list(per_page=1000)
        for project in projects:
            logger.info('PROJECT: "%s/%s"', group.path, project.path)
            try:
                mirror_project(group, project, wiki=False)
            except Exception as e:
                logger.error(str(e))
            try:
                mirror_project(group, project, wiki=True)
            except Exception as e:
                logger.error(str(e))


if __name__ == '__main__':
    # main()

  Github: Checking out pull requests locally

  • Find the ID number of the pull request. This is the sequence of digits right after the pull request's title.

  • Fetch the reference to the pull request based on its ID number, creating a new branch in the process:

    $ git fetch origin pull/ID/head:BRANCHNAME
    
  • Switch to the new branch that's based on this pull request:

    $ git checkout BRANCHNAME
    
  • At this point, you can do anything you want with this branch. You can run some local tests, or merge other branches into the branch.

When you're ready, you can push the new branch up:

$ git push origin BRANCHNAME

References: Modifying an inactive pull request locally

  django: how to filter out GET static and media messages with logging

# "django: how to filter out GET static and media messages with logging?"
# https://stackoverflow.com/questions/23833642/django-how-to-filter-out-get-static-and-media-messages-with-logging#41620949
def skip_static_requests(record):
    if record.args[0].startswith('GET /static/'):  # filter whatever you want
        return False
    return True

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        # use Django's built in CallbackFilter to point to your filter
        'skip_static_requests': {
            '()': 'django.utils.log.CallbackFilter',
            'callback': skip_static_requests
        }
    },
    'formatters': {
        # django's default formatter
        'django.server': {
            '()': 'django.utils.log.ServerFormatter',
            'format': '[%(server_time)s] %(message)s',
        }
    },
    'handlers': {
        # django's default handler...
        'django.server': {
            'level': 'INFO',
            'filters': ['skip_static_requests'],  # <- ...with one change
            'class': 'logging.StreamHandler',
            'formatter': 'django.server',
        },
    },
    'loggers': {
        # django's default logger
        'django.server': {
            'handlers': ['django.server'],
            'level': 'INFO',
            'propagate': False,
        },
    }
}

  elapsed_time():: Human readable time interval between two values of time.time()

import math
import time

def elapsed_time(t0, t1, with_milliseconds=False):
    """
    Human readable time interval between two values of time.time()
    """
    milliseconds, seconds = math.modf(t1 - t0)
    dt = time.strftime("%H:%M:%S", time.gmtime(seconds))
    if with_milliseconds:
        dt += ('%.3f' % milliseconds)[1:]
    return dt

  Symmetrical editing fo a M2M relation in Django admin

Given:

class Team(models.Model):

    name = models.CharField(_('name'), max_length=150, unique=True)
    jobs = models.ManyToManyField(Job, verbose_name=u'Jobs', blank=True,
        related_name='teams')


@admin.register(Team)
class TeamAdmin(admin.ModelAdmin):
    filter_horizontal = ['jobs',]
    ...

Django already provides a widget for editing the relation in the Team change form.

To have a similar behaviour in the Job change form as well, proceed as follows:

from django import forms
from django.contrib.admin.widgets import FilteredSelectMultiple


class JobAdminForm(forms.ModelForm):
    teams = forms.ModelMultipleChoiceField(
        queryset=Team.objects.all(),
        required=False,
        widget=FilteredSelectMultiple(
            verbose_name=_('Teams'),
            is_stacked=False
        )
    )

    class Meta:
        model = Job
        exclude = []

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        if self.instance and self.instance.pk:
            self.fields['teams'].initial = self.instance.teams.all()

    def save(self, commit=True):
        job = super().save(commit=commit)
        if commit:
            job.teams = self.cleaned_data['teams']
        else:
            old_save_m2m = self.save_m2m
            def new_save_m2m():
                old_save_m2m()
                job.teams.set(self.cleaned_data['teams'])
            self.save_m2m = new_save_m2m
        return job


@admin.register(Job)
class JobAdmin(BaseModelAdmin):
    form = JobAdminForm
    ...

Similarly, you can even add multiple M2M relation to the JobAdminForm:

class JobAdminForm(forms.ModelForm):
    users = forms.ModelMultipleChoiceField(
        queryset=User.objects.all(),
        required=False,
        widget=FilteredSelectMultiple(
            verbose_name=_('Users'),
            is_stacked=False
        )
    )
    teams = forms.ModelMultipleChoiceField(
        queryset=Team.objects.all(),
        required=False,
        widget=FilteredSelectMultiple(
            verbose_name=_('Teams'),
            is_stacked=False
        )
    )

    class Meta:
        model = Job
        exclude = []

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        if self.instance and self.instance.pk:
            self.fields['users'].initial = self.instance.users.all()
            self.fields['teams'].initial = self.instance.teams.all()

    def save(self, commit=True):
        job = super().save(commit=commit)
        if commit:
            job.users = self.cleaned_data['users']
            job.teams = self.cleaned_data['teams']
        else:
            old_save_m2m = self.save_m2m
            def new_save_m2m():
                old_save_m2m()
                job.users.set(self.cleaned_data['users'])
                job.teams.set(self.cleaned_data['teams'])
            self.save_m2m = new_save_m2m
        return job

  Sort a list of dictionaries in Python3

import functools


def sort_results(data):
    """
    Sort on "anno", than on "label"
    """

    def mycmp(a, b):
        if a < b:
            return -1
        elif a > b:
            return 1
        return 0

    def compare(x, y):
        if x['anno'] == y['anno']:
            return mycmp(x['label'], y['label'])
        return mycmp(x['anno'], y['anno'])

    data.sort(key=functools.cmp_to_key(compare), reverse=False)

  Prevent Mac from falling asleep

The caffeinate command in terminal keeps the Mac from falling asleep.

For indefinite:

caffeinate

For a specific time, eg 5 hours:

caffeinate -i -t 18000

  How to Install and upgrade Node.js and NPM on Ubuntu

Installation (on Ubuntu 18)

sudo apt-get install nodejs
sudo apt-get install npm

Results:

$ which node
/usr/bin/node
$ which nodejs
/usr/bin/nodejs
$ which npm
/usr/bin/npm
$ node -v
v8.10.0
$ nodejs -v
v8.10.0
$ npm -v
3.5.2

Upgrade (on Ubuntu 18)

npm install -g n
n stable

Results:

$ which node
/usr/local/bin/node
$ which nodejs
/usr/bin/nodejs
$ which npm
/usr/local/bin/npm
$ node -v
v14.16.0
$ nodejs -v
v8.10.0
$ npm -v
6.14.11

  cmp: Replacement for built-in function cmp that was removed in Python 3

As part of the move away from cmp-style comparisons, the cmp() function was removed in Python 3.

If it is necessary (usually to conform to an external API), you can provide it with this code:

def cmp(x, y):
    """
    Replacement for built-in function cmp that was removed in Python 3

    Compare the two objects x and y and return an integer according to
    the outcome. The return value is negative if x < y, zero if x == y
    and strictly positive if x > y.
    """

    return (x > y) - (x < y)

Credits:

https://portingguide.readthedocs.io/en/latest/comparisons.html#the-cmp-function

  Network Configuration in Debian 10

Preliminary downloads

Create the following VMs:

  • debian1 (root/debian1, master/debian1)
  • debian2 (root/debian2, master/debian2)

How to Install VirtualBox Guest Additions on Debian 10 Linux:

https://linuxize.com/post/how-to-install-virtualbox-guest-additions-on-debian-10/

Additional settings:

apt install net-tools
apt install sudo

echo 'export PATH=/usr/sbin:$PATH' >> ~/.bashrc


View current network configuration

$ ip a

or

$ ifconfig

To find out the DNS servers IPs:

$ cat /etc/resolv.conf


Change network configuration (temporary)

Basic network configuration includes setting:

  • a static or dynamic IP address
  • adding a gateway
  • adding DNS server information

The following commands will change network settings; however, the new settings will not be permanent: once you reboot your system, the settings will be removed.

1. Assign an IP address to the interface

sudo ifconfig <interface> <IP_address> netmask <subnetmask> up

example:

sudo ifconfig eth0 192.168.72.165 netmask 255.255.255.0 up

2. Set the Default Gateway

sudo route add default gw <IP_address> <interface>

example:

sudo route add default gw 192.168.72.2 eth0

3. Set Your DNS server

echo "nameserver <IP_address>" > /etc/resolv.conf

example:

echo "nameserver 8.8.8.8" > /etc/resolv.conf

4. Remove IP address from a network interface

ip address del <IP_address> dev <interface>

Once done, you can test your configuration by running the ifconfig command as follows:

sudo ifconfig -a


Change network settings permanently by using the interfaces file

Add a static address (file `/etc/network/interfaces`):

auto eth0

# static IP address
iface eth0 inet static
    address 192.168.72.165
    netmask 255.255.255.0
    gateway 192.168.72.2

Please note that the address, netmask and gateway line must start with leading whitespace!

Dynamically assign the address (file `/etc/network/interfaces`):

auto eth0
iface eth0 inet dhcp

Defining the (DNS) Nameservers

To add DNS server information, we will need to edit the /etc/resolv.conf file.

Example:

nameserver 8.8.8.8
nameserver 192.168.72.2

Reload network settings

ifdown eth0
ifup eth0


Configurazione DHCP server

sudo apt-get update
sudo apt-get install isc-dhcp-server

cat /etc/default/isc-dhcp-server

    INTERFACESv4="enp0s10"

cat /etc/dhcp/dhcpd.conf

    default-lease-time 600;
    max-lease-time 7200;

    subnet 192.168.100.64 netmask 255.255.255.224 {
        range 192.168.100.70 192.168.100.80;
        option routers 192.168.100.94;
        option domain-name-servers 8.8.8.8 8.8.4.4;
        option domain-name "acme.mo.it";

        #host TOR-datacenterclient {
        #    hardware ethernet 08:00:27:27:3a:1c;
        #    fixed-address 192.168.100.65;
        #}

    }

systemctl stop isc-dhcp-server
systemctl start isc-dhcp-server

tail -f /var/log/syslog


Comandi utili

ip addr
route
route -n
ip route


  Django & HTTPS tricks

Testing HTTPS in development

sudo pip install django-sslserver

INSTALLED_APPS = [
    ...
    "sslserver",
    ...
]

then:

python manage.py runsslserver 0.0.0.0:8888

Note that for some reason port 8000 hangs; use 8888 instead.

Credits: https://stackoverflow.com/questions/8023126/how-can-i-test-https-connections-with-django-as-easily-as-i-can-non-https-connec#51384868

With NGINX

In your Django settings, set the SECURE_PROXY_SSL_HEADER setting:

SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')

Then, you need Nginx to set the custom header in the reverse proxy. In the Nginx site settings:

location / {
    # ...
    proxy_set_header X-Forwarded-Proto $scheme;
}

This way request.scheme == 'https' and request.is_secure() returns True. request.build_absolute_uri() returns https://... and so on...

Credits: https://stackoverflow.com/questions/8153875/how-to-deploy-an-https-only-site-with-django-nginx#19637196

  PostgreSQL ODBC connection from Ubuntu

Install the basic config tools for the UNIX ODBC:

sudo apt-get install unixodbc unixodbc-dev

Install ODBC drivers for PostgreSQL:

sudo apt-get install odbc-postgresql

Configure the PostgreSQL and ODBC driver.

File /etc/odbcinst.ini:

[PostgreSQL ANSI]
Description=PostgreSQL ODBC driver (ANSI version)
Driver=psqlodbca.so
Setup=libodbcpsqlS.so
Debug=0
CommLog=1
UsageCount=1

[PostgreSQL Unicode]
Description=PostgreSQL ODBC driver (Unicode version)
Driver=psqlodbcw.so
Setup=libodbcpsqlS.so
Debug=0
CommLog=1
UsageCount=1

File /etc/odbc.ini:

[Gallery]
Driver = PostgreSQL Unicode
Description = ODBC connection to Gallery via PostgreSQL Unicode
Trace = No
#Trace = Yes
#TraceFile = /tmp/sql.log
Servername = localhost
Port = 5432
#Protocol = 8.4
Database = gallery
UserName = ****
Password = ****
#ReadOnly = No

Test the ODBC to PostgreSQL connection by running the isql command, which reads the /etc/odbc.ini file:

$ isql -v gallery <username> <password>
+---------------------------------------+
| Connected!                            |
|                                       |
| sql-statement                         |
| help [tablename]                      |
| quit                                  |
|                                       |
+---------------------------------------+

SQL> select count(*) from users_user;
+---------------------+
| count               |
+---------------------+
| 3                   |
+---------------------+
SQLRowCount returns 1
1 rows fetched

  CookiesDirective for Privacy Policy

In your base.html add:

<script src="{% static 'js/jquery.cookiesdirective.js' %}"></script>

...

<script>

    $(document).ready(function() {

        /*
        $.cookiesDirective({
            privacyPolicyUri: 'my-privacy-policy.html',
            explicitConsent: false
        });

        $(".privacyPolicyButton").click(function (e) {
            e.preventDefault();
            $("#privacyPolicy").modal('show');
        });
        */


        // Cookie setting script wrapper
        var cookieScripts = function () {
            /*
            // Internal javascript called
            console.log("Running");

            // Loading external javascript file
            $.cookiesDirective.loadScript({
                uri:'external.js',
                appendTo: 'eantics'
            });
            */

            $("#privacyPolicyTag").animate({
              opacity:'1'
            },2000);
        };

        $.cookiesDirective({
            privacyPolicyUri: '/pc-policy/privacy-policy/',
            explicitConsent: false,
            scriptWrapper: cookieScripts,
            position: 'bottom',
            message: '{% trans "This site uses cookies. More details can be found in our" %}',
            privacyPolicyTitle: '{% trans "Privacy Policy" %}',
            dismissButtonTitle: '{% trans "Do not show this message again" %}'
        });

        /*
        $(".privacyPolicyButton").click(function (e) {
            e.preventDefault();
            //$("#privacyPolicyDialog").modal('show');
            $("#privacyPolicyDialog").dialog({
                width: 500,
                height: 400
            });
        });
        */

    });

</script>

where jquery.cookiesdirective.js has been adapted from: https://github.com/krazyjakee/CookiesDirective

/* Cookies Directive - The rewrite. Now a jQuery plugin
 * Version: 2.0.1
 * Author: Ollie Phillips
 * 24 October 2013
 *
 * More information at http://cookiesdirective.com
 */

;(function($) {
  $.cookiesDirective = function(options) {

    // Default Cookies Directive Settings
    var settings = $.extend({
      //Options
      explicitConsent: true,
      position: 'top',
      duration: 10,
      limit: 0,
      message: null,
      dismissButtonTitle: 'Do not show this message again',
      privacyPolicyTitle: 'Privacy Policy',
      cookieScripts: null,
      privacyPolicyUri: 'privacy.html',
      scriptWrapper: function(){},
      // Styling
      fontFamily: 'helvetica',
      fontColor: '#FFFFFF',
      fontSize: '13px',
      backgroundColor: '#000000',
      backgroundOpacity: '80',
      //linkColor: '#CA0000',
      linkColor: '#ff0',
      impliedSubmitColor: '#000'
    }, options);

    // Perform consent checks
    if(!getCookie('cookiesDirective')) {
      if(settings.limit > 0) {
        // Display limit in force, record the view
        if(!getCookie('cookiesDisclosureCount')) {
          setCookie('cookiesDisclosureCount',1,1);
        } else {
          var disclosureCount = getCookie('cookiesDisclosureCount');
          disclosureCount ++;
          setCookie('cookiesDisclosureCount',disclosureCount,1);
        }

        // Have we reached the display limit, if not make disclosure
        if(settings.limit >= getCookie('cookiesDisclosureCount')) {
          disclosure(settings);
        }
      } else {
        // No display limit
        disclosure(settings);
      }

      // If we don't require explicit consent, load up our script wrapping function
      if(!settings.explicitConsent) {
        settings.scriptWrapper.call();
      }
    } else {
      // Cookies accepted, load script wrapping function
      settings.scriptWrapper.call();
    }
  };

  // Used to load external javascript files into the DOM
  $.cookiesDirective.loadScript = function(options) {
    var settings = $.extend({
      uri:    '',
      appendTo:   'body'
    }, options);

    var elementId = String(settings.appendTo);
    var sA = document.createElement("script");
    sA.src = settings.uri;
    sA.type = "text/javascript";
    sA.onload = sA.onreadystatechange = function() {
      if ((!sA.readyState || sA.readyState == "loaded" || sA.readyState == "complete")) {
        return;
      }
    };
    switch(settings.appendTo) {
      case 'head':
        $('head').append(sA);
          break;
      case 'body':
        $('body').append(sA);
          break;
      default:
        $('#' + elementId).append(sA);
    }
  };

  // Helper scripts
  // Get cookie
  var getCookie = function(name) {
    var nameEQ = name + "=";
    var ca = document.cookie.split(';');
    for(var i=0;i < ca.length;i++) {
      var c = ca[i];
      while (c.charAt(0)==' ') c = c.substring(1,c.length);
      if (c.indexOf(nameEQ) === 0) return c.substring(nameEQ.length,c.length);
    }
    return null;
  };

  // Set cookie
  var setCookie = function(name,value,days) {
    var expires = "";
    if (days) {
      var date = new Date();
      date.setTime(date.getTime()+(days*24*60*60*1000));
      expires = "; expires="+date.toGMTString();
    }
    document.cookie = name+"="+value+expires+"; path=/";
  };

  // Detect IE < 9
  var checkIE = function(){
    var version;
    if (navigator.appName == 'Microsoft Internet Explorer') {
          var ua = navigator.userAgent;
          var re = new RegExp("MSIE ([0-9]{1,}[\\.0-9]{0,})");
          if (re.exec(ua) !== null) {
              version = parseFloat(RegExp.$1);
      }
      if (version <= 8.0) {
        return true;
      } else {
        if(version == 9.0) {
          if(document.compatMode == "BackCompat") {
            // IE9 in quirks mode won't run the script properly, set to emulate IE8
            var mA = document.createElement("meta");
            mA.content = "IE=EmulateIE8";
            document.getElementsByTagName('head')[0].appendChild(mA);
            return true;
          } else {
            return false;
          }
        }
        return false;
      }
      } else {
      return false;
    }
  };

  // Disclosure routines
  var disclosure = function(options) {
    var settings = options;
    settings.css = 'fixed';

    // IE 9 and lower has issues with position:fixed, either out the box or in compatibility mode - fix that
    if(checkIE()) {
      settings.position = 'top';
      settings.css = 'absolute';
    }

    // Any cookie setting scripts to disclose
    var scriptsDisclosure = '';
    if (settings.cookieScripts) {
      var scripts = settings.cookieScripts.split(',');
      var scriptsCount = scripts.length;
      var scriptDisclosureTxt = '';
      if(scriptsCount>1) {
        for(var t=0; t < scriptsCount - 1; t++) {
           scriptDisclosureTxt += scripts[t] + ', ';
        }
        scriptsDisclosure = ' We use ' +  scriptDisclosureTxt.substring(0,  scriptDisclosureTxt.length - 2) + ' and ' + scripts[scriptsCount - 1] + ' scripts, which all set cookies. ';
      } else {
        scriptsDisclosure = ' We use a ' + scripts[0] + ' script which sets cookies.';
      }
    }

    // Create overlay, vary the disclosure based on explicit/implied consent
    // Set our disclosure/message if one not supplied
    var html = '';
    html += '<div id="epd">';
    html += '<div id="cookiesdirective" style="position:'+ settings.css +';'+ settings.position + ':-300px;left:0px;width:100%;';
    html += 'height:auto;background:' + settings.backgroundColor + ';opacity:.' + settings.backgroundOpacity + ';';
    html += '-ms-filter: “alpha(opacity=' + settings.backgroundOpacity + ')”; filter: alpha(opacity=' + settings.backgroundOpacity + ');';
    html += '-khtml-opacity: .' + settings.backgroundOpacity + '; -moz-opacity: .' + settings.backgroundOpacity + ';';
    html += 'color:' + settings.fontColor + ';font-family:' + settings.fontFamily + ';font-size:' + settings.fontSize + ';';
    html += 'text-align:center;z-index:1000;">';
    html += '<div style="position:relative;height:auto;width:90%;padding:10px;margin-left:auto;margin-right:auto;">';

    if(!settings.message) {
      if(settings.explicitConsent) {
        // Explicit consent message
        // settings.message = 'This site uses cookies. Some of the cookies we ';
        // settings.message += 'use are essential for parts of the site to operate and have already been set.';

        settings.message = "Questo sito o gli strumenti terzi da questo utilizzati si avvalgono di cookie necessari al funzionamento ed utili alle finalità illustrate nella cookie policy.<br />Chiudendo questo banner, scorrendo questa pagina, cliccando su un link o proseguendo la navigazione in altra maniera, acconsenti all’uso dei cookie.";

      } else {
        // Implied consent message
        //settings.message = 'We have placed cookies on your computer to help make this website better.';
        settings.message = "Questo sito o gli strumenti terzi da questo utilizzati si avvalgono di cookie necessari al funzionamento ed utili alle finalità illustrate nella cookie policy.<br />Chiudendo questo banner, scorrendo questa pagina, cliccando su un link o proseguendo la navigazione in altra maniera, acconsenti all’uso dei cookie.";
      }
    }
    html += settings.message;

    // Build the rest of the disclosure for implied and explicit consent
    /* !!!
    if(settings.explicitConsent) {
      // Explicit consent disclosure
      html += scriptsDisclosure + 'You may delete and block all cookies from this site, but parts of the site will not work.';
      html += 'To find out more about cookies on this website, see our <a style="color:'+ settings.linkColor + ';font-weight:bold;';
      html += 'font-family:' + settings.fontFamily + ';font-size:' + settings.fontSize + ';" href="'+ settings.privacyPolicyUri + '">privacy policy</a>.<br/>';
      html += '<div id="epdnotick" style="color:#ca0000;display:none;margin:2px;"><span style="background:#cecece;padding:2px;">You must tick the "I accept cookies from this site" box to accept</span></div>';
      html += '<div style="margin-top:5px;">I accept cookies from this site <input type="checkbox" name="epdagree" id="epdagree" />&nbsp;';
      html += '<input type="submit" name="explicitsubmit" id="explicitsubmit" value="Continue"/><br/></div></div>';



    } else {
      // Implied consent disclosure
      //html += scriptsDisclosure + ' More details can be found in our <a style="color:'+ settings.linkColor + ';';
      html += scriptsDisclosure + '<br />Ulteriori informazioni sono specificate nella nostra <a class="privacyPolicyButton" style="color:'+ settings.linkColor + ';';
      html += 'font-weight:bold;font-family:' + settings.fontFamily + ';font-size:' + settings.fontSize + ';" href="'+ settings.privacyPolicyUri + '">Privacy Policy</a>.';
      //html += '<div style="margin-top:5px;"><input type="submit" name="impliedsubmit" id="impliedsubmit" value="Do not show this message again"/></div></div>';
      html += '<div style="margin-top:5px;"><input type="submit" name="impliedsubmit" id="impliedsubmit" style="color:'+ settings.impliedSubmitColor +';" ' + 'value="Non mostrare più questo messaggio"/></div></div>';
    }
    */

    html += ' <a class="privacyPolicyButton" style="color:#ff0;font-weight:bold;font-family:helvetica;font-size:13px;" href="' + settings.privacyPolicyUri + '">' +
        settings.privacyPolicyTitle + '</a>';
    html += '<div style="margin-top:5px;"><input type="submit" name="impliedsubmit" id="impliedsubmit" value="' + settings.dismissButtonTitle + '"/></div></div>';

    html += '</div></div>';
    $('body').append(html);

    // Serve the disclosure, and be smarter about branching
    var dp = settings.position.toLowerCase();
    if(dp != 'top' && dp!= 'bottom') {
      dp = 'top';
    }
    var opts = { in: null, out: null};
    if(dp == 'top') {
      opts.in = {'top':'0'};
      opts.out = {'top':'-300'};
    } else {
      opts.in = {'bottom':'0'};
      opts.out = {'bottom':'-300'};
    }

    // Start animation
    $('#cookiesdirective').animate(opts.in, 1000, function() {
      // Set event handlers depending on type of disclosure
      if(settings.explicitConsent) {
        // Explicit, need to check a box and click a button
        $('#explicitsubmit').click(function() {
          if($('#epdagree').is(':checked')) {
            // Set a cookie to prevent this being displayed again
            setCookie('cookiesDirective',1,365);
            // Close the overlay
            $('#cookiesdirective').animate(opts.out,1000,function() {
              // Remove the elements from the DOM and reload page
              $('#cookiesdirective').remove();
              location.reload(true);
            });
          } else {
            // We need the box checked we want "explicit consent", display message
            $('#epdnotick').css('display', 'block');
          }
        });
      } else {
        // Implied consent, just a button to close it
        $('#impliedsubmit').click(function() {
          // Set a cookie to prevent this being displayed again
          setCookie('cookiesDirective',1,365);
          // Close the overlay
          $('#cookiesdirective').animate(opts.out,1000,function() {
            // Remove the elements from the DOM and reload page
            $('#cookiesdirective').remove();
          });
        });
      }

      // Set a timer to remove the warning after 'settings.duration' seconds
      setTimeout(function(){
        $('#cookiesdirective').animate({
          opacity:'0'
        },2000, function(){
          $('#cookiesdirective').css(dp,'-300px');
        });
        $("#privacyPolicyTag").animate({
          opacity:'1'
        },2000);
      }, settings.duration * 1000);
    });
  };
})(jQuery);

  Docker notes


OBSOLETE! see https://github.com/morlandi/DockerDev

Cheatsheet

docker container ls -a List all containers
docker container ls -a -q List all containers in "quiet" mode
docker rm $(docker container ls -a -q) Remove all containsers
docker image ls -a List images
docker rmi IMAGENAME Remove image

Running an instantaneous Ubuntu Machine

System Message: WARNING/2 (<string>, line 23)

Title underline too short.

Running an instantaneous Ubuntu Machine
--------------------------------------

We will run a Ubuntu machine working with an interactive docker container.

This is the very ground level knowledge on how using Docker as a user environment, rather than a standalone container with an app running in it.

docker run --interactive --tty ubuntu /bin/bash

What we did:

  • we created a ubuntu machine with a random name

or:

docker run --name ubuntu -v /Users/morlandi/tmp/sources:/src -t -i ubuntu /bin/bash

What we did:

  • we created a ubuntu machine named "ubuntu"
  • with a volumes attached to it, directly from our machine
  • we mapped the volume using the "-v" flag [local]:[destination],

The lifetime of a container is the life time of its single main process.

Time to check docker, see what happened with it:

docker ps -a

CONTAINER ID        IMAGE               COMMAND             NAMES
e41423f7856b        ubuntu              "/bin/bash"         ubuntu
ed5887209364        ubuntu              "/bin/bash"         cool_proskuriakova

Finally, we cleanup the images:

docker rm -f cool_proskuriakova
docker rm -f ubuntu

Mac docker volume mount using osxfs

Docker Desktop for Mac started using osxfs for supporting volume mounting on MacOS.

The following command mounts the ~/Desktop directory to the docker container:

docker run -it -v ~/Desktop:/Desktop ubuntu bash

Proof:

from the container:

root@71f6e65abc6e:/# ls /
bin  boot  Desktop  dev  etc  home  lib  lib64  media  mnt  opt  proc  root  run  sbin  srv  sys  tmp  usr  var

root@71f6e65abc6e:/# ls -l /Desktop/

    -rw-r--r-- 1 root root 127138 Jul 17 14:52 '/Desktop/Screenshot 2020-07-17 at 16.51.58.png'
    -rw-r--r-- 1 root root 648435 Jul 17 15:47 '/Desktop/Screenshot 2020-07-17 at 17.47.21.png'
    -rw-r--r-- 1 root root 249486 Jul 17 16:15 '/Desktop/Screenshot 2020-07-17 at 18.15.15.png'
    -rw-r--r-- 1 root root 821404 Jul 18 08:57 '/Desktop/Screenshot 2020-07-18 at 10.57.38.png'

from the host:

$ ls -l ~/Desktop

    -rw-r--r--@ 1 morlandi  staff  127138 Jul 17 16:52 Desktop/Screenshot 2020-07-17 at 16.51.58.png
    -rw-r--r--@ 1 morlandi  staff  648435 Jul 17 17:47 Desktop/Screenshot 2020-07-17 at 17.47.21.png
    -rw-r--r--@ 1 morlandi  staff  249486 Jul 17 18:15 Desktop/Screenshot 2020-07-17 at 18.15.15.png
    -rw-r--r--@ 1 morlandi  staff  821404 Jul 18 10:57 Desktop/Screenshot 2020-07-18 at 10.57.38.png

  django-imagekit: Generate predictable filenames in cache

file settings.py:

#
#  ImageKit
#

#IMAGEKIT_DEFAULT_CACHEFILE_STRATEGY = 'imagekit.cachefiles.strategies.Optimistic'

# Generate predictable filenames in cache
#IMAGEKIT_SPEC_CACHEFILE_NAMER = 'imagekit.cachefiles.namers.source_name_as_path'
IMAGEKIT_SPEC_CACHEFILE_NAMER = 'main.imagekit_overrides.source_name_as_path'

where file main.imagekit_overrides.py is:

# adapted from "imagekit.cachefiles.namers.source_name_as_path.py"

from django.conf import settings
import os
from imagekit.utils import format_to_extension, suggest_extension


def my_get_hash(generator):
    name = ''
    try:
        processor = generator.processors[0]
        width = processor.width
        height = processor.height
        name = '%dx%d' % (width, height)
    except Exception as e:
        name = generator.get_hash()
    return name


def source_name_as_path(generator):
    """
    A namer that, given the following source file name::

        photos/thumbnails/bulldog.jpg

    will generate a name like this::

        /path/to/generated/images/photos/thumbnails/bulldog/5ff3233527c5ac3e4b596343b440ff67.jpg

    where "/path/to/generated/images/" is the value specified by the
    ``IMAGEKIT_CACHEFILE_DIR`` setting.

    """
    source_filename = getattr(generator.source, 'name', None)

    if source_filename is None or os.path.isabs(source_filename):
        # Generally, we put the file right in the cache file directory.
        dir = settings.IMAGEKIT_CACHEFILE_DIR
    else:
        # For source files with relative names (like Django media files),
        # use the source's name to create the new filename.
        dir = os.path.join(settings.IMAGEKIT_CACHEFILE_DIR,
                           os.path.splitext(source_filename)[0])

    ext = suggest_extension(source_filename or '', generator.format)
    #return os.path.normpath(os.path.join(dir, '%s%s' % (generator.get_hash(), ext)))
    return os.path.normpath(os.path.join(dir, '%s%s' % (my_get_hash(generator), ext)))

  Send an HTTP request using Telnet

$ telnet brainstorm.it 80
Trying 116.203.5.135...
Connected to brainstorm.it.
Escape character is '^]'.

GET / HTTP/1.0
Host: brainstorm.it
HTTP/1.1 301 Moved Permanently
Server: nginx
Date: Tue, 07 Jul 2020 06:48:29 GMT
Content-Type: text/html
Content-Length: 178
Connection: close
Location: https://brainstorm.it/

<html>
<head><title>301 Moved Permanently</title></head>
<body bgcolor="white">
<center><h1>301 Moved Permanently</h1></center>
<hr><center>nginx</center>
</body>
</html>
Connection closed by foreign host.

For https-only reachable url:

$ ncat --ssl brainstorm.it 443
GET / HTTP/1.0
Host: brainstorm.it
...
...
                });
            });
        })(jQuery);
    </script>

  </body>
</html>

  Building Python 3 on embedded Linux

curl -O https://www.python.org/ftp/python/3.8.4/Python-3.8.4rc1.tgz
tar xzvf Python-3.8.4rc1.tgz
./configure
make
make install

References:

Install virtualenvwrapper (optional)

pip3 install virtualenvwrapper

Add this to ~/.profile:

alias ll='/bin/ls -lh'

export VIRTUALENVWRAPPER_PYTHON=/usr/local/bin/python3
export WORKON_HOME=~/.virtualenvs
#mkdir -p $WORKON_HOME
source /usr/local/bin/virtualenvwrapper.sh

To create a new virtualenv:

mkvirtualenv webapp --seeder=pip

  django-parler: Make existing fields translatable

Procedura per traduzione tabella Product

  • I campi name, slug e description (pre-esistenti) devono essere tradotti

  • Commentato ProductAdmin (sostituito con pass)

  • Rinominati i campi field --> old_field in Product

  • Creata migrazione "product_rename_old_field"

  • Product derivato da TranslatableBaseModel anziche' BaseModel

  • introdotti i campi "translations":

    translations = TranslatedFields(
        name = models.CharField(max_length=1024, verbose_name=_("Name")),
        slug = models.SlugField(_("Slug"), max_length=255, unique=True),
        description = models.TextField(blank=True, verbose_name=_("Description")),
    
        meta = {'unique_together': [('language_code', 'slug')]},
    )
    
  • Creata migrazione "product_rename_add_translation_fields"

  • Fix migrazione precedente:

    migrations.CreateModel(
        name='Product',
        ...
        bases=(parler.models.TranslatableModelMixin, models.Model),
    ),
    
  • Creata data migration "product_migrate_translatable_fields":

    from django.db import migrations
    from django.conf import settings
    
    
    def forwards_func(apps, schema_editor):
        MyModel = apps.get_model('backend', 'Product')
        MyModelTranslation = apps.get_model('backend', 'ProductTranslation')
    
        for object in MyModel.objects.all():
            MyModelTranslation.objects.create(
                master_id=object.pk,
                language_code=settings.LANGUAGE_CODE,
                name=object.old_name,
                slug=object.old_slug,
                description=object.old_description,
            )
    
    
    class Migration(migrations.Migration):
    
        dependencies = [
            ('backend', '0003_product_add_translation_fields'),
        ]
    
        operations = [
            migrations.RunPython(forwards_func),
        ]
    
  • Commentati i campi "old_field" e creata migrazione "product_remove_old_fields"

  • ProductAdmin derivata da TranslatableBaseModelAdmin anziche' BaseModelAdmin

  • In ProductAdmin, utilizzato metodo get_prepopulated_fields() anziche' attributo prepopulated_fields

  • Eseguite migrazioni

  django-imagekit: How to cleanup cache backend and remove generated thumbnails from storage

Best effort to regenerate the thumbnail when the associated image is updated

In a django Model, I keep an ImageField "image" and an associated "thumbnail", based on from imagekit.models.ImageSpecField.

I might need to rebuild the image, keeping the same filename for the bitmap.

Obviously, the thumbnail needs to be updated as well; my first attempt was:

self.thumbnail.generate(force=True)

but that's not enough; despite the force=True parameter, the thumbnail remains unchanged.

After some trials and errors, I ended up with the following workaround:

class MyModel(model.Model):

    ...

    image = models.ImageField(_('Image'), null=True, blank=True, upload_to=get_acquisition_media_file_path)
    thumbnail = ImageSpecField(source='image', processors=[ResizeToFill(200, 100)], format='PNG')

    ...

    def save_image_from_data(self):

        # Create a new random image
        from random import randint
        rgb = (randint(0, 255), randint(0, 255), randint(0, 255))
        image_size = (200, 200)
        image = PILImage.new('RGB', image_size, rgb)
        with io.BytesIO() as output:
            image.save(output, format="PNG")
            contents = output.getvalue()

        # Best effort to remove the thumbnail, if any
        try:
            file = self.thumbnail.file
        #except FileNotFoundError:
        except:
            pass
        try:
            cache_backend = self.thumbnail.cachefile_backend
            cache_backend.cache.delete(cache_backend.get_key(file))
            #self.thumbnail.storage.delete(file)
            self.thumbnail.storage.delete(file.name)
        except:
            pass

        # Save image
        self.image = ContentFile(contents, "acquisition_%d.png" % self.position)
        self.save()

        # Regenerate thumbnail
        self.thumbnail.generate(force=True)

        return True

Thumbnails cleanup

ImageKit has a bug where files are cached and not deleted right away:

https://github.com/matthewwithanm/django-imagekit/issues/229#issuecomment-315690575

An interesting solution to the is the following.

UNTESTED: I didn't try this yet in a real situation

from django.db.models.signals import post_delete

    @staticmethod
    def post_delete(sender, instance, **kwargs):
        for field in ['thumbnail', ]:
            field = getattr(instance, field)
            try:
                file = field.file
            except FileNotFoundError:
                pass
            else:
                cache_backend = field.cachefile_backend
                cache_backend.cache.delete(cache_backend.get_key(file))
                field.storage.delete(file)
        instance.file.delete(save=False)

post_delete.connect(MyModel.post_delete, sender=MyModel)

References

Manually clear the cache

because in ImageKit 4.0 django cache backend is configured to preserve the state of images forever then you need to clear it also.

from imagekit.utils import get_cache
get_cache().clear()

which just runs "FLUSHDB" on Redis !