fidzu : Django

The first Beta was released today, so it seems a good day to start the mirror for the 2.0 branch of Django. For the record, main purposes of this mirror are: being a lightweight read-only repository to clone from for production servers hide the ugly git stuff behind a great mercurial interface The clone is […]

17 Oct 2017 11:22pm GMT

The Mailchimp API is extensive...

17 Oct 2017 7:35pm GMT

key_name = template.add_parameter(Parameter(
    "KeyName",
    Description="Name of an existing EC2 KeyPair to enable SSH access to "
                "the AWS EC2 instances",
    Type="AWS::EC2::KeyPair::KeyName",
    ConstraintDescription="must be the name of an existing EC2 KeyPair."
))

dokku_version = template.add_parameter(Parameter(
    "DokkuVersion",
    Description="Dokku version to install, e.g., \"v0.10.4\" (see "
                "https://github.com/dokku/dokku/releases).",
    Type="String",
    Default="v0.10.4",
))

dokku_web_config = template.add_parameter(Parameter(
    "DokkuWebConfig",
    Description="Whether or not to enable the Dokku web config "
                "(defaults to false for security reasons).",
    Type="String",
    AllowedValues=["true", "false"],
    Default="false",
))

dokku_vhost_enable = template.add_parameter(Parameter(
    "DokkuVhostEnable",
    Description="Whether or not to use vhost-based deployments "
                "(e.g., foo.domain.name).",
    Type="String",
    AllowedValues=["true", "false"],
    Default="true",
))

root_size = template.add_parameter(Parameter(
    "RootVolumeSize",
    Description="The size of the root volume (in GB).",
    Type="Number",
    Default="30",
))

ssh_cidr = template.add_parameter(Parameter(
    "SshCidr",
    Description="CIDR block from which to allow SSH access. Restrict "
                "this to your IP, if possible.",
    Type="String",
    Default="0.0.0.0/0",
))

template.add_mapping('RegionMap', {
    "ap-northeast-1": {"AMI": "ami-0417e362"},
    "ap-northeast-2": {"AMI": "ami-536ab33d"},
    "ap-south-1": {"AMI": "ami-df413bb0"},
    "ap-southeast-1": {"AMI": "ami-9f28b3fc"},
    "ap-southeast-2": {"AMI": "ami-bb1901d8"},
    "ca-central-1": {"AMI": "ami-a9c27ccd"},
    "eu-central-1": {"AMI": "ami-958128fa"},
    "eu-west-1": {"AMI": "ami-674cbc1e"},
    "eu-west-2": {"AMI": "ami-03998867"},
    "sa-east-1": {"AMI": "ami-a41869c8"},
    "us-east-1": {"AMI": "ami-1d4e7a66"},
    "us-east-2": {"AMI": "ami-dbbd9dbe"},
    "us-west-1": {"AMI": "ami-969ab1f6"},
    "us-west-2": {"AMI": "ami-8803e0f0"},
})

instance_role = iam.Role(
     "ContainerInstanceRole",
     template=template,
     AssumeRolePolicyDocument=dict(Statement=[dict(
         Effect="Allow",
         Principal=dict(Service=["ec2.amazonaws.com"]),
         Action=["sts:AssumeRole"],
     )]),
     Path="/",
     Policies=[
         assets_management_policy,  # defined in assets.py
         logging_policy,  # defined in logs.py
     ]
 )

 instance_profile = iam.InstanceProfile(
     "ContainerInstanceProfile",
     template=template,
     Path="/",
     Roles=[Ref(instance_role)],
 )

security_group = template.add_resource(ec2.SecurityGroup(
    'SecurityGroup',
    GroupDescription='Allows SSH access from SshCidr and HTTP/HTTPS '
                     'access from anywhere.',
    VpcId=Ref(vpc),
    SecurityGroupIngress=[
        ec2.SecurityGroupRule(
            IpProtocol='tcp',
            FromPort=22,
            ToPort=22,
            CidrIp=Ref(ssh_cidr),
        ),
        ec2.SecurityGroupRule(
            IpProtocol='tcp',
            FromPort=80,
            ToPort=80,
            CidrIp='0.0.0.0/0',
        ),
        ec2.SecurityGroupRule(
            IpProtocol='tcp',
            FromPort=443,
            ToPort=443,
            CidrIp='0.0.0.0/0',
        ),
    ]
))

eip = template.add_resource(ec2.EIP("Eip"))

ec2_instance_name = 'Ec2Instance'
ec2_instance = template.add_resource(ec2.Instance(
    ec2_instance_name,
    ImageId=FindInMap("RegionMap", Ref("AWS::Region"), "AMI"),
    InstanceType=container_instance_type,
    KeyName=Ref(key_name),
    SecurityGroupIds=[Ref(security_group)],
    IamInstanceProfile=Ref(instance_profile),
    SubnetId=Ref(container_a_subnet),
    BlockDeviceMappings=[
        ec2.BlockDeviceMapping(
            DeviceName="/dev/sda1",
            Ebs=ec2.EBSBlockDevice(
                VolumeSize=Ref(root_size),
            )
        ),
    ],
    # ...
    Tags=Tags(
        Name=Ref("AWS::StackName"),
    ),
)

ec2_instance = template.add_resource(ec2.Instance(
    # ...
    CreationPolicy=CreationPolicy(
        ResourceSignal=ResourceSignal(
            Timeout='PT10M',  # 10 minutes
        ),
    ),
    # ...
)

ec2_instance = template.add_resource(ec2.Instance(
    # ...
    UserData=Base64(Join('', [
        '#!/bin/bash\n',
        # install cfn helper scripts
        'apt-get update\n',
        'apt-get -y install python-pip\n',
        'pip install https://s3.amazonaws.com/cloudformation-examples/'
        'aws-cfn-bootstrap-latest.tar.gz\n',
        'cp /usr/local/init/ubuntu/cfn-hup /etc/init.d/cfn-hup\n',
        'chmod +x /etc/init.d/cfn-hup\n',
        # don't start cfn-hup yet until we install cfn-hup.conf
        'update-rc.d cfn-hup defaults\n',
        # call our "on_first_boot" configset (defined below):
        'cfn-init --stack="', Ref('AWS::StackName'), '"',
        ' --region=', Ref('AWS::Region'),
        ' -r %s -c on_first_boot\n' % ec2_instance_name,
        # send the exit code from cfn-init to our CreationPolicy:
        'cfn-signal -e $? --stack="', Ref('AWS::StackName'), '"',
        ' --region=', Ref('AWS::Region'),
        ' --resource %s\n' % ec2_instance_name,
    ])),
    # ...
)

ec2_instance = template.add_resource(ec2.Instance(
    # ...
    Metadata=cloudformation.Metadata(
        cloudformation.Init(
            cloudformation.InitConfigSets(
                on_first_boot=['install_dokku', 'set_dokku_env', 'start_cfn_hup'],
                on_metadata_update=['set_dokku_env'],
            ),
            install_dokku=cloudformation.InitConfig(
                commands={
                    '01_fetch': {
                        'command': Join('', [
                            'wget https://raw.githubusercontent.com/dokku/dokku/',
                            Ref(dokku_version),
                            '/bootstrap.sh',
                        ]),
                        'cwd': '~',
                    },
                    '02_install': {
                        'command': 'sudo -E bash bootstrap.sh',
                        'env': {
                            'DOKKU_TAG': Ref(dokku_version),
                            'DOKKU_VHOST_ENABLE': Ref(dokku_vhost_enable),
                            'DOKKU_WEB_CONFIG': Ref(dokku_web_config),
                            'DOKKU_HOSTNAME': domain_name,
                            # use the key configured by key_name
                            'DOKKU_KEY_FILE': '/home/ubuntu/.ssh/authorized_keys',
                            # should be the default, but be explicit just in case
                            'DOKKU_SKIP_KEY_FILE': 'false',
                        },
                        'cwd': '~',
                    },
                },
            ),
            set_dokku_env=cloudformation.InitConfig(
                commands={
                    '01_set_env': {
                        # redirect output to /dev/null so we don't write
                        # environment variables to log file
                        'command': 'dokku config:set --global {} >/dev/null'.format(
                            ' '.join(['=$'.join([k, k]) for k in dict(environment_variables).keys()]),
                        ),
                        'env': dict(environment_variables),
                    },
                },
            ),
            start_cfn_hup=cloudformation.InitConfig(
                commands={
                    '01_start': {
                        'command': 'service cfn-hup start',
                    },
                },
                files={
                    '/etc/cfn/cfn-hup.conf': {
                        'content': Join('', [
                            '[main]\n',
                            'stack=', Ref('AWS::StackName'), '\n',
                            'region=', Ref('AWS::Region'), '\n',
                            'umask=022\n',
                            'interval=1\n',  # check for changes every minute
                            'verbose=true\n',
                        ]),
                        'mode': '000400',
                        'owner': 'root',
                        'group': 'root',
                    },
                    '/etc/cfn/hooks.d/cfn-auto-reloader.conf': {
                        'content': Join('', [
                            # trigger the on_metadata_update configset on any
                            # changes to Ec2Instance metadata
                            '[cfn-auto-reloader-hook]\n',
                            'triggers=post.update\n',
                            'path=Resources.%s.Metadata\n' % ec2_instance_name,
                            'action=/usr/local/bin/cfn-init',
                            ' --stack=', Ref('AWS::StackName'),
                            ' --resource=%s' % ec2_instance_name,
                            ' --configsets=on_metadata_update',
                            ' --region=', Ref('AWS::Region'), '\n',
                            'runas=root\n',
                        ]),
                        'mode': '000400',
                        'owner': 'root',
                        'group': 'root',
                    },
                },
            ),
        ),
    ),
    # ...
)

ssh dokku@<your domain or IP> apps:create python-sample
git clone https://github.com/heroku/python-sample.git
cd python-sample
git remote add dokku dokku@<your domain or IP>:python-sample
git push dokku master

16 Oct 2017 6:30pm GMT

I received an email asking how to structure your Django Project if you are using an API.

It's common for people to want to know the best practices for structuring your code with your Django project.

People think that maybe if I can structure my project correctly, it would be easier to do X.

Sometimes your .py files can get so out of hand that you might want to break them up into smaller files.

Your API is just another Django app

But, How do you organize your API views and URLs?

You can do it pretty simply.

When I create an API using Django REST Framework, I'll put my API code in it's separate app and put all the views, URLs, Serializers, etc. inside that one directory. Everything is exactly the same.

I'll update my URLs so that each route to my API starts with something like /api/v1/

Steps to creating an API

1. Go into the directory with manage.py
2. Run command to create a new app for your new API: python manage.py startapp api
3. Update the URLConf and include the URLs for your API: url(r'^api/v1/', include('api.urls', namespace="api")),
4. Add some files in your api folder: api_views.py, serializers.py, etc.
5. Then, sign up for my Free Django REST Framework Email course to learn how to setup your own API.

16 Oct 2017 9:00am GMT

Introduction

Welcome to the last part of our tutorial series! In this tutorial, we are going to deploy our Django application to a production server. We are also going to configure an Email service and HTTPS certificates for our servers.

At first, I thought about given an example using a Virtual Private Server (VPS), which is more generic and then using one Platform as a Service such as Heroku. But it was too much detail, so I ended up creating this tutorial focused on VPSs.

Our project is live! If you want to check online before you go through the text, this is the application we are going to deploy: www.djangoboards.com.

Version Control

Version control is an extremely important topic in software development. Especially when working with teams and maintaining production code at the same time, several features are being developed in parallel. No matter if it's a one developer project or a multiple developers project, every project should use version control.

There are several options of version control systems out there. Perhaps because of the popularity of GitHub, Git become the de facto standard in version control. So if you are not familiar version control, Git is a good place to start. There are many tutorials, courses, and resources in general so that it's easy to find help.

GitHub and Code School have a great interactive tutorial about Git, which I used years ago when I started moving from SVN to Git. It's a very good introduction.

This is such an important topic that I probably should have brought it up since the first tutorial. But the truth is I wanted the focus of this tutorial series to be on Django. If all this is new for you, don't worry. It's important to take one step at a time. Your first project won't be perfect. It's important to keep learning and evolving your skills slowly but with constancy.

A very good thing about Git is that it's much more than just a version control system. There's a rich ecosystem of tools and services built around it. Some good examples are continuous integration, deployment, code review, code quality, and project management.

Using Git to support the deployment process of Django projects works very well. It's a convenient way to pull the latest version from the source code repository or to rollback to a specific version in case of a problem. There are many services that integrate with Git so to automate test execution and deployment for example.

If you don't have Git installed on your local machine, grab the installed from https://git-scm.com/downloads.

Basic Setup

First thing, set your identity:

git config --global user.name "Vitor Freitas"
git config --global user.email vitor@simpleisbetterthancomplex.com

In the project root (the same directory as manage.py is), initialize a git repository:

git init

Initialized empty Git repository in /Users/vitorfs/Development/myproject/.git/

Check the status of the repository:

git status

On branch master

Initial commit

Untracked files:
  (use "git add <file>..." to include in what will be committed)

  accounts/
  boards/
  manage.py
  myproject/
  requirements.txt
  static/
  templates/

nothing added to commit but untracked files present (use "git add" to track)

Before we proceed in adding the source files, create a new file named .gitignore in the project root. This special file will help us keep the repository clean, without unnecessary files like cache files or logs for example.

You can grab a generic .gitignore file for Python projects from GitHub.

Make sure to rename it from Python.gitignore to just .gitignore (the dot is important!).

You can complement the .gitignore file telling it to ignore SQLite database files for example:

.gitignore

__pycache__/
*.py[cod]
.env
venv/


# SQLite database files

*.sqlite3

Now add the files to the repository:

git add .

Notice the dot here. The command above is telling Git to add all untracked files within the current directory.

Now make the first commit:

git commit -m "Initial commit"

Always write a comment telling what this commit is about, briefly describing what have you changed.

Remote Repository

Now let's setup GitHub as a remote repository. First, create a free account on GitHub, then confirm your email address. After that, you will be able to create public repositories.

For now, just pick a name for the repository, don't initialize it with a README, or add a .gitignore or add a license so far. Make sure you start the repository empty:

After you create the repository you should see something like this:

Now let's configure it as our remote repository:

git remote add origin git@github.com:sibtc/django-boards.git

Now push the code to the remote server, that is, to the GitHub repository:

git push origin master

Counting objects: 84, done.
Delta compression using up to 4 threads.
Compressing objects: 100% (81/81), done.
Writing objects: 100% (84/84), 319.70 KiB | 0 bytes/s, done.
Total 84 (delta 10), reused 0 (delta 0)
remote: Resolving deltas: 100% (10/10), done.
To git@github.com:sibtc/django-boards.git
 * [new branch]      master -> master

I create this repository just to demonstrate the process to create a remote repository with an existing code base. The source code of the project is officially hosted in this repository: https://github.com/sibtc/django-beginners-guide.

Project Settings

No matter if the code is stored in a public or private remote repository, sensitive information should never be committed and pushed to the remote repository. That includes secret keys, passwords, API keys, etc.

At this point, we have to deal with two specific types of configuration in our settings.py module:

• Sensitive information such as keys and passwords;
• Configurations that are specific to a given environment.

Passwords and keys can be stored in environment variables or using local files (not committed to the remote repository):

# environment variables
import os
SECRET_KEY = os.environ['SECRET_KEY']

# or local files
with open('/etc/secret_key.txt') as f:
    SECRET_KEY = f.read().strip()

For that, there's a great utility library called Python Decouple that I use in every single Django project I develop. It will search for a local file named .env to set the configuration variables and will fall back to the environment variables. It also provides an interface to define default values, transform the data into int, bool, and list when applicable.

It's not mandatory, but I really find it a very useful tool. And it works like a charm with services like Heroku.

First, let's install it:

pip install python-decouple

myproject/settings.py

from decouple import config

SECRET_KEY = config('SECRET_KEY')

Now we can place the sensitive information in a special file named .env (notice the dot in front) in the same directory where the manage.py file is:

myproject/
 |-- myproject/
 |    |-- accounts/
 |    |-- boards/
 |    |-- myproject/
 |    |-- static/
 |    |-- templates/
 |    |-- .env        <-- here!
 |    |-- .gitignore
 |    |-- db.sqlite3
 |    +-- manage.py
 +-- venv/

.env

SECRET_KEY=rqr_cjv4igscyu8&&(0ce(=sy=f2)p=f_wn&@0xsp7m$@!kp=d

The .env file is ignored in the .gitignore file, so every time we are going to deploy the application or run in a different machine, we will have to create a .env file and add the necessary configuration.

Now let's install another library to help us write the database connection in a single line. This way it's easier to write different database connection strings in different environments:

pip install dj-database-url

For now, all the configurations we will need to decouple:

myproject/settings.py

from decouple import config, Csv
import dj_database_url

SECRET_KEY = config('SECRET_KEY')
DEBUG = config('DEBUG', default=False, cast=bool)
ALLOWED_HOSTS = config('ALLOWED_HOSTS', cast=Csv())
DATABASES = {
    'default': dj_database_url.config(
        default=config('DATABASE_URL')
    )
}

Example of a .env file for our local machine:

SECRET_KEY=rqr_cjv4igscyu8&&(0ce(=sy=f2)p=f_wn&@0xsp7m$@!kp=d
DEBUG=True
ALLOWED_HOSTS=.localhost,127.0.0.1

Notice that in the DEBUG configuration we have a default, so in production we can ignore this configuration because it will be set to False automatically, as it is supposed to be.

Now the ALLOWED_HOSTS will be transformed into a list like ['.localhost', '127.0.0.1'. ]. Now, this is on our local machine, for production we will set it to something like ['.djangoboards.com', ] or whatever domain you have.

This particular configuration make sure your application is only served to this domain.

Tracking Requirements

It's a good practice to keep track of the project's dependencies, so to be easier to install it in another machine.

We can check the currently installed Python libraries by running the command:

pip freeze

dj-database-url==0.4.2
Django==1.11.6
django-widget-tweaks==1.4.1
Markdown==2.6.9
python-decouple==3.1
pytz==2017.2

Create a file named requirements.txt in the project root, and add the dependencies there:

requirements.txt

dj-database-url==0.4.2
Django==1.11.6
django-widget-tweaks==1.4.1
Markdown==2.6.9
python-decouple==3.1

I kept the pytz==2017.2 out, because it is automatically installed by Django.

You can update your source code repository:

git add .
git commit -m "Add requirements.txt file"
git push origin master

Domain Name

If we are going to deploy a Django application properly, we will need a domain name. It's important to have a domain name to serve the application, configure an email service and configure an https certificate.

Lately I've been using Namecheap a lot. You can get a .com domain for $8.88/year or if you are just trying things out, you could register a .xyz domain for $0.99/year.

Anyway, you are free to use any registrar. For demonstrate the deployment process I registered the www.DjangoBoards.com domain.

Deployment Strategy

Here is an overview of the deployment strategy we are going to use in this tutorial:

The cloud is our Virtual Private Server provided by Digital Ocean. You can sign up to Digital Ocean using my affiliate link to get a free $10 credit (only valid for new accounts).

Upfront we will have NGINX, illustrated by the ogre. NGINX will receive all requests to the server. But it won't try to do anything smart if the request data. All it is going to do is decide if the requested information is a static asset that he can serve by himself, or if it's something more complicated. If so, it will pass the request to Gunicorn.

The NGINX will also be configured with HTTPS certificates. Meaning it will only accept requests via HTTPS. If the client tries to make a request via HTTP, NGINX will first redirect the user to the HTTPS and only then it will decide what do to with the request.

We are also going to install this certbot to automatically renew the Let's Encrypt certificates.

Gunicorn is an application server. Depending on the number of processors the server has, it can spawn multiple workers to process multiple requests in parallel. It manage the workload and execute the Python and Django code.

Django is the one doing the hard work. It may access the database (PostgreSQL) or the file system. But for the most part, the work is done inside the views, rendering templates, all those things that we've been coding for the past weeks. After Django process the request, it returns a response back to Gunicorn, who returns the result to NGINX that will finally delivery the response to the client.

We are also going to install PostgreSQL, a production quality database system. Because of Django's ORM system it's easy to switch databases.

The last step is to install Supervisor. Supervisor is a process control system and it will keep an eye on Gunicorn and Django to make sure everything runs smoothly. If the server restarts, or if Gunicorn crashes, it will automatically restart it.

Deploying to a VPS (Digital Ocean)

You may use any other VPS (Virtual Private Server) you like. The configuration should be very similar, after all, we are going to use Ubuntu 16.04 as our server.

First let's create a new server (on Digital Ocean they call it "Droplet"). Select Ubuntu 16.04:

Pick the size. The smallest droplet is enough:

Then choose a hostname for your droplet (in my case "django-boards"):

If you have a SSH key you can add it to your account. Then you will be able to log in the server using it. Otherwise they will email you the root password.

Now pick the server's IP address:

Before we log in to the server, let's point our domain name to this IP address. This will save some time because DNS settings usually takes a few minutes to propagate.

So here we basically added two A records, one pointing to the naked domain "djangoboards.com" and the other one for "www.djangoboards.com". We will use NGINX to configure a canonical URL.

Not let's log in to the server using your terminal:

ssh root@45.55.144.54
root@45.55.144.54's password:

Then you should see the following message:

You are required to change your password immediately (root enforced)
Welcome to Ubuntu 16.04.3 LTS (GNU/Linux 4.4.0-93-generic x86_64)

 * Documentation:  https://help.ubuntu.com
 * Management:     https://landscape.canonical.com
 * Support:        https://ubuntu.com/advantage

  Get cloud support with Ubuntu Advantage Cloud Guest:
    http://www.ubuntu.com/business/services/cloud

0 packages can be updated.
0 updates are security updates.


Last login: Sun Oct 15 18:39:21 2017 from 82.128.188.51
Changing password for root.
(current) UNIX password:

Set the new password, and let's start to configure the server.

sudo apt-get update
sudo apt-get -y upgrade

If you get any prompt during the upgrade, select the option "keep the local version currently installed".

Python 3.6

sudo add-apt-repository ppa:deadsnakes/ppa
sudo apt-get update
sudo apt-get install python3.6

PostgreSQL

sudo apt-get -y install postgresql postgresql-contrib

NGINX

sudo apt-get -y install nginx

Supervisor

sudo apt-get -y install supervisor

sudo systemctl enable supervisor
sudo systemctl start supervisor

Virtualenv

wget https://bootstrap.pypa.io/get-pip.py
sudo python3.6 get-pip.py
sudo pip3.6 install virtualenv

Application User

Create a new user with the command below:

adduser boards

Usually I just pick the name of the application. Enter a password and optionally add some extra info to the prompt.

Now add the user to the sudoers list:

gpasswd -a boards sudo

PostgreSQL Database Setup

First switch to the postgres user:

sudo su - postgres

Create a database user:

createuser u_boards

Create a new database and set the user as the owner:

createdb django_boards --owner u_boards

Define a strong password for the user:

psql -c "ALTER USER u_boards WITH PASSWORD 'BcAZoYWsJbvE7RMgBPzxOCexPRVAq'"

We can now exit the postgres user:

exit

Django Project Setup

Switch to the application user:

sudo su - boards

First we can check where we are:

pwd
/home/boards

First let's clone the repository with our code:

git clone https://github.com/sibtc/django-beginners-guide.git

Start a virtual environment:

virtualenv venv -p python3.6

Initialize the virtualenv:

source venv/bin/activate

Install the requirements:

pip install -r django-beginners-guide/requirements.txt

We will have to add two extra libraries here, the Gunicorn and the PostgreSQL driver:

pip install gunicorn
pip install psycopg2

Now inside the /home/boards/django-beginners-guide folder, let's create a .env file to store the database credentials, the secret key and everything else:

/home/boards/django-beginners-guide/.env

SECRET_KEY=rqr_cjv4igscyu8&&(0ce(=sy=f2)p=f_wn&@0xsp7m$@!kp=d
ALLOWED_HOSTS=.djangoboards.com
DATABASE_URL=postgres://u_boards:BcAZoYWsJbvE7RMgBPzxOCexPRVAq@localhost:5432/django_boards

Here is the syntax of the database URL: postgres://db_user:db_password@db_host:db_port/db_name.

Now let's migrate the database, collect the static files and create a super user:

cd django-beginners-guide

python manage.py migrate

Operations to perform:
  Apply all migrations: admin, auth, boards, contenttypes, sessions
Running migrations:
  Applying contenttypes.0001_initial... OK
  Applying auth.0001_initial... OK
  Applying admin.0001_initial... OK
  Applying admin.0002_logentry_remove_auto_add... OK
  Applying contenttypes.0002_remove_content_type_name... OK
  Applying auth.0002_alter_permission_name_max_length... OK
  Applying auth.0003_alter_user_email_max_length... OK
  Applying auth.0004_alter_user_username_opts... OK
  Applying auth.0005_alter_user_last_login_null... OK
  Applying auth.0006_require_contenttypes_0002... OK
  Applying auth.0007_alter_validators_add_error_messages... OK
  Applying auth.0008_alter_user_username_max_length... OK
  Applying boards.0001_initial... OK
  Applying boards.0002_auto_20170917_1618... OK
  Applying boards.0003_topic_views... OK
  Applying sessions.0001_initial... OK

Now the static files:

python manage.py collectstatic

Copying '/home/boards/django-beginners-guide/static/js/jquery-3.2.1.min.js'
Copying '/home/boards/django-beginners-guide/static/js/popper.min.js'
Copying '/home/boards/django-beginners-guide/static/js/bootstrap.min.js'
Copying '/home/boards/django-beginners-guide/static/js/simplemde.min.js'
Copying '/home/boards/django-beginners-guide/static/css/app.css'
Copying '/home/boards/django-beginners-guide/static/css/bootstrap.min.css'
Copying '/home/boards/django-beginners-guide/static/css/accounts.css'
Copying '/home/boards/django-beginners-guide/static/css/simplemde.min.css'
Copying '/home/boards/django-beginners-guide/static/img/avatar.svg'
Copying '/home/boards/django-beginners-guide/static/img/shattered.png'
...

This command copy all the static assets to an external directory where NGINX can serve the files for us. More on that later.

Now create a super user for the application:

python manage.py createsuperuser

Configuring Gunicorn

So, Gunicorn is the one responsible for executing the Django code behind a proxy server.

Create a new file named gunicorn_start inside /home/boards:

#!/bin/bash

NAME="django_boards"
DIR=/home/boards/django-beginners-guide
USER=boards
GROUP=boards
WORKERS=3
BIND=unix:/home/boards/run/gunicorn.sock
DJANGO_SETTINGS_MODULE=myproject.settings
DJANGO_WSGI_MODULE=myproject.wsgi
LOG_LEVEL=error

cd $DIR
source ../venv/bin/activate

export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE
export PYTHONPATH=$DIR:$PYTHONPATH

exec ../venv/bin/gunicorn ${DJANGO_WSGI_MODULE}:application \
  --name $NAME \
  --workers $WORKERS \
  --user=$USER \
  --group=$GROUP \
  --bind=$BIND \
  --log-level=$LOG_LEVEL \
  --log-file=-

This script will start the application server. We are providing some information such as where the Django project is, which application user to be used to run the server, and so on.

Now make this file executable:

chmod u+x gunicorn_start

Create two empty folders, one for the socket file and one to store the logs:

mkdir run logs

Right now the directory structure inside /home/boards should look like this:

django-beginners-guide/
gunicorn_start
logs/
run/
staticfiles/
venv/

The staticfiles folder was created by the collectstatic command.

Configuring Supervisor

First create an empty log file inside the /home/boards/logs/ folder:

touch logs/gunicorn.log

Now create a new supervisor file:

sudo vim /etc/supervisor/conf.d/boards.conf

[program:boards]
command=/home/boards/gunicorn_start
user=boards
autostart=true
autorestart=true
redirect_stderr=true
stdout_logfile=/home/boards/logs/gunicorn.log

Save the file and run the commands below:

sudo supervisorctl reread
sudo supervisorctl update

Now check the status:

sudo supervisorctl status boards

boards                           RUNNING   pid 308, uptime 0:00:07

Configuring NGINX

Next step is to set up the NGINX server to serve the static files and to pass the requests to Gunicorn:

Add a new configuration file named boards inside /etc/nginx/sites-available/:

upstream app_server {
    server unix:/home/boards/run/gunicorn.sock fail_timeout=0;
}

server {
    listen 80;
    server_name www.djangoboards.com;  # here can also be the IP address of the server

    keepalive_timeout 5;
    client_max_body_size 4G;

    access_log /home/boards/logs/nginx-access.log;
    error_log /home/boards/logs/nginx-error.log;

    location /static/ {
        alias /home/boards/staticfiles/;
    }

    # checks for static file, if not found proxy to app
    location / {
        try_files $uri @proxy_to_app;
    }

    location @proxy_to_app {
      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
      proxy_set_header Host $http_host;
      proxy_redirect off;
      proxy_pass http://app_server;
    }
}

Create a symbolic link to the sites-enabled folder:

sudo ln -s /etc/nginx/sites-available/boards /etc/nginx/sites-enabled/boards

Remove the default NGINX website:

sudo rm /etc/nginx/sites-enabled/default

Restart the NGINX service:

sudo service nginx restart

At this point, if the DNS have already propagated, the website should be available on the URL www.djangoboards.com.

Configuring an Email Service

One of the best options to get started is Mailgun. It offers a very reliable free plan covering 12,000 emails per month.

First sign up for a free account. Then just follow the steps, it's very straightforward. You will have to work together with the service you registered your domain. In my case, it was Namecheap.

First click on add domain to add a new domain to your account. Follow the instructions and make sure you use "mg." subdomain:

Now grab the first set of DNS records, it's two TXT records:

Add it to your domain, using the web interface offered by your registrar:

Do the same thing with the MX records:

Add them to the domain:

Now this step is not mandatory, but since we are already here, confirm it as well:

After adding all the DNS records, click in the Check DNS Records Now button:

Now we need to have some patience. Sometimes it takes a while to validate the DNS.

Meanwhile we can configure the application to receive the connection parameters.

myproject/settings.py

EMAIL_BACKEND = config('EMAIL_BACKEND', default='django.core.mail.backends.smtp.EmailBackend')
EMAIL_HOST = config('EMAIL_HOST', default='')
EMAIL_PORT = config('EMAIL_PORT', default=587, cast=int)
EMAIL_HOST_USER = config('EMAIL_HOST_USER', default='')
EMAIL_HOST_PASSWORD = config('EMAIL_HOST_PASSWORD', default='')
EMAIL_USE_TLS = config('EMAIL_USE_TLS', default=True, cast=bool)

DEFAULT_FROM_EMAIL = 'Django Boards <noreply@djangoboards.com>'
EMAIL_SUBJECT_PREFIX = '[Django Boards] '

Then, my local machine .env file would look like this:

SECRET_KEY=rqr_cjv4igscyu8&&(0ce(=sy=f2)p=f_wn&@0xsp7m$@!kp=d
DEBUG=True
ALLOWED_HOSTS=.localhost,127.0.0.1
DATABASE_URL=sqlite:///db.sqlite3
EMAIL_BACKEND=django.core.mail.backends.console.EmailBackend

And my production .env file would look like this:

SECRET_KEY=rqr_cjv4igscyu8&&(0ce(=sy=f2)p=f_wn&@0xsp7m$@!kp=d
ALLOWED_HOSTS=.djangoboards.com
DATABASE_URL=postgres://u_boards:BcAZoYWsJbvE7RMgBPzxOCexPRVAq@localhost:5432/django_boards
EMAIL_HOST=smtp.mailgun.org
EMAIL_HOST_USER=postmaster@mg.djangoboards.com
EMAIL_HOST_PASSWORD=ED2vmrnGTM1Rdwlhazyhxxcd0F

You can find your credentials in the Domain Information section on Mailgun.

• EMAIL_HOST: SMTP Hostname
• EMAIL_HOST_USER: Default SMTP Login
• EMAIL_HOST_PASSWORD: Default Password

We can test the new settings in the production server. Make the changes in the settings.py file in your local machine, commit the changes to the remote repository. Then, in the server pull the new code and restart the Gunicorn process:

git pull

Edit the .env file with the email credentials.

Then restart the Gunicorn process:

sudo supervisorctl restart boards

Now we can try to start the password reset process:

On the Mailgun dashboard you can have some statistics about the email delivery:

Configuring HTTPS Certificate

Now let's protect our application with a nice HTTPS certificate provided by Let's Encrypt.

Setting up HTTPS has never been that easy. And better, we can get it for free nowadays. They provide a solution called certbot which takes care of installing and renewing the certificates for us. It's very straightforward:

sudo apt-get update
sudo apt-get install software-properties-common
sudo add-apt-repository ppa:certbot/certbot
sudo apt-get update
sudo apt-get install python-certbot-nginx

Now install the certs:

sudo certbot --nginx

Just follow the prompts. When asked about:

Please choose whether or not to redirect HTTP traffic to HTTPS, removing HTTP access.

Choose 2 to redirect all HTTP traffic to HTTPS.

With that the site is already being served over HTTPS:

Setup the auto renew of the certs. Run the command below to edit the crontab file:

sudo crontab -e

Add the following line to the end of the file:

0 4 * * * /usr/bin/certbot renew --quiet

This command will run everyday at 4am. All certificates expiring within 30 days will automatically be renewed.

Conclusions

Thanks a lot for all those who followed this tutorial series, giving comments and feedback! I really appreciate! This was the last tutorial of the series. I hope you enjoyed it!

Even though this was the last part of the tutorial series, I plan to write a few follow-up tutorials exploring other interesting topics as well, such as database optimization and adding more features on top of what we have at the moment.

By the way, if you are interested in contributing to the project, few free to submit pull requests! The source code of the project is available on GitHub: https://github.com/sibtc/django-beginners-guide/

And please let me know what else you would like to see next! :-)

← Part 6 - Class-Based Views

Tutorial Series Index →

16 Oct 2017 5:00am GMT

You asked for it, you got it. Two Scoops of Django 1.11 now comes in Kindle Mobi and ePub formats. If you've already purchased the PDF, you'll be receiving an email containing a new download link that adds the .mobi and .epub files. Otherwise, get yours at the following links:

• PDF/Kindle/ePub versions available here
• Amazon (Kindle only)

12 Oct 2017 11:56am GMT

As of PostgreSQL 9.5 we have UPSERT support. Technically, it's ON CONFLICT, but it's basically a way to execute an UPDATE statement in case the INSERT triggers a conflict on some column value. By the way, here's a great blog post that demonstrates how to use ON CONFLICT.

In this Django app I have a model that has a field called hash which has a unique=True index on it. What I want to do is either insert a row, or if the hash is already in there, it should increment the count and the modified_at timestamp instead.

The Code(s)

Here's the basic version in "pure Django ORM":

if MissingSymbol.objects.filter(hash=hash_).exists():
    MissingSymbol.objects.filter(hash=hash_).update(
        count=F('count') + 1,
        modified_at=timezone.now()
    )
else:
    MissingSymbol.objects.create(
        hash=hash_,
        symbol=symbol,
        debugid=debugid,
        filename=filename,
        code_file=code_file or None,
        code_id=code_id or None,
    )

Here's that same code rewritten in "pure SQL":

from django.db import connection


with connection.cursor() as cursor:
    cursor.execute("""
        INSERT INTO download_missingsymbol (
            hash, symbol, debugid, filename, code_file, code_id,
            count, created_at, modified_at
        ) VALUES (
            %s, %s, %s, %s, %s, %s,
            1, CLOCK_TIMESTAMP(), CLOCK_TIMESTAMP()
          )
        ON CONFLICT (hash)
        DO UPDATE SET
            count = download_missingsymbol.count + 1,
            modified_at = CLOCK_TIMESTAMP()
        WHERE download_missingsymbol.hash = %s
        """, [
            hash_, symbol, debugid, filename,
            code_file or None, code_id or None,
            hash_
        ]
    )

Both work.

Note the use of CLOCK_TIMESTAMP() instead of NOW(). Since Django wraps all writes in transactions if you use NOW() it will be evaluated to the same value for the whole transaction, thus making unit testing really hard.

But which is fastest?

The Results

First of all, this hard to test locally because my Postgres is running locally in Docker so the network latency in talking to a network Postgres means that the latency is less and having to do two different executions would cost more if the network latency is more.

I ran a simple benchmark where it randomly picked one of the two code blocks (above) depending on a 50% chance.
The results are:

METHOD     MEAN       MEDIAN
SQL        6.99ms     6.61ms
ORM        10.28ms    9.86ms

So doing it with a block of raw SQL instead is 1.5 times faster. But this difference would surely grow when the network latency is higher.

Discussion

There's an alternative and that's to use django-postgres-extra but I'm personally hesitant. The above little raw SQL hack is the only thing I need and adding more libraries makes far-future maintenance harder.

Beyond the time optimization of being able to send only 1 SQL instruction to PostgreSQL, the biggest benefit is avoiding concurrency race conditions. From the documentation:

"ON CONFLICT DO UPDATE guarantees an atomic INSERT or UPDATE outcome; provided there is no independent error, one of those two outcomes is guaranteed, even under high concurrency. This is also known as UPSERT - "UPDATE or INSERT"."

I'm going to keep this little hack. It's not beautiful but it works and saves time and gives me more comfort around race conditions.

11 Oct 2017 7:57pm GMT

One of Django's nice "batteries included" features is the ability to send emails when an error is encountered. This is a great feature for small sites where minor problems would otherwise go unnoticed.

Once your site start getting lots of traffic, however, the feature turns into a liability. An error might fire off thousands of emails in rapid succession. Not only will this put extra load on your web servers, but you could also take down (or get banned from) your email server in the process.

One of the first things you want to do when setting up a high-traffic Django site is replace the default error email functionality with an error reporting service like Sentry. Once you've got Sentry setup, what's the best way to disable error emails? Unfortunately, there isn't one simple answer. Let's dig into why...

How does Django Send the Emails?

First let's look at how Django sends those emails. If you look at the source, you'll find it is defined in the default logging config using standard Python logging:

'django': {
        'handlers': ['console', 'mail_admins'],
        'level': 'INFO',
    }

As you can guess, the mail_admins handler is the one that does the work. It is defined as:

'mail_admins': {
        'level': 'ERROR',
        'filters': ['require_debug_false'],
        'class': 'django.utils.log.AdminEmailHandler'
    }

In plain English, this says that any log messages that are ERROR or higher in the django Python module will get passed to django.utils.log.AdminEmailHandler when the require_debug_false filter evaluates to True.

Here is an example of the default Django logging tree.

Disabling the Logger

Simply removing mail_admins from the handlers should be easy, right? Unfortunately, the answer involves diving into the idiosyncracies of Python logging configuration.

If you search around the internet, you'll come up with a few different options, but rarely an explanation of why they work or the potential drawbacks.

⛔️ Option 1: disable_existing_loggers

One option is to set disable_existing_loggers to True in your LOGGING setting. This parameter is the source of a lot of confusion and the results of using it can be unintuitive. From the Python docs:

The default is True because this enables old behavior in a backward-compatible way. This behaviour is to disable any existing loggers unless they or their ancestors are explicitly named in the logging configuration.

See the effect this has on the logging tree here. Note: all loggers are flagged as "Disabled".

The issue with this approach is that if you later define a logger for a submodule of django, say, django.db. As per the docs, you'll find the default django logger is now re-enabled, mail_admins handler and all.

That's pretty unintuitive behavior and for that reason, I don't recommend this approach. When setting up logging, always set disable_existing_loggers to False to avoid this issue.

⛔️ Option 2: Redefine the Logger

There is a second option which is a little more surgical than option #2. By redefining the logger for django, we will replace the handlers that are defined for it in the default config. Here's a full logging config which will accomplish that:

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'console': {
            'level': 'INFO',
            'class': 'logging.StreamHandler',
        },
    },
    'loggers': {
        # Redefining the logger for the `django` module
        # prevents invoking the `AdminEmailHandler`
        'django': {
            'handlers': ['console'],
            'level': 'INFO',
        },
    }
}

See the effect this has on the logging tree here.

Unfortunately, this change wipes out any handlers set on child loggers. One such potentially helpful handler is the request logging used by Django's runserver command. For that reason, I don't recommend this approach either.

✅ Option 3: Copy the Default Logger

If you'd like to keep Django's default config and just remove the mail_admins handler, your best bet is to simply modify the default dictionary directly.

from copy import deepcopy
from django.utils.log import DEFAULT_LOGGING

logging_dict = deepcopy(DEFAULT_LOGGING)
logging_dict['loggers']['django']['handlers'] = ['console']
LOGGING = logging_dict

See the effect this has on the logging tree here.

This accomplishes the goal, but also introduces some fragility into the code. A change to DEFAULT_LOGGING in a future version of Django could break this code. In practice, you may also find you want to make more changes to the default config (for example, logging to console when DEBUG = False).

✅ Option 4: LOGGING_CONFIG = None

Setting LOGGING_CONFIG = None in your settings prevents Django from setting up any logging at all. This is the nuclear option.

import logging.config
LOGGING_CONFIG = None
logging.config.dictConfig({
    "version": 1,
    "disable_existing_loggers": False,
})

See the effect this has on the logging tree here.

This is a valid approach and probably the best one to take once you want any sort of fine-grained control over your logging. You wouldn't want to use this exact config in practice, but instead, take it as a starting point for building your own custom configuration to suit the needs of your project.

In the process of writing this, I discovered a few "features" in Python logging that surprised me. Brandon Rhode's logging_tree package was invaluable for peeking under the hood after the Django initialized the logging tree. I strongly recommend dropping it into your project to verify logging is setup as you expect.

While this post focuses on the mail_admins functionality, in the real-world, you'll want a more robust logging config (including a catch-all root logger, and a logger for your project module), but we'll leave that for another post.

If you have any Django logging tips, we'd love to hear them!

10 Oct 2017 9:27pm GMT

Analytics can make our service...

10 Oct 2017 7:39pm GMT

When checking out the nodes of your Kubernetes cluster, you may have noticed some containers called "pause" running when you do a `docker ps` on the node. $ docker ps CONTAINER ID IMAGE COMMAND ... ... 3b45e983c859 gcr.io/google_containers/pause-amd64:3.0 "/pause" ... ... dbfc35b00062 gcr.io/google_containers/pause-amd64:3.0 "/pause" ... ... c4e998ec4d5d gcr.io/google_containers/pause-amd64:3.0 "/pause" ... ... 508102acf1e7 gcr.io/google_containers/pause-amd64:3.0 "/pause[...]

10 Oct 2017 7:40am GMT

Introduction

Welcome to the sixth part of the tutorial series! In this tutorial, we are going to explore in great detail the Class-Based Views. We are also going to refactor some of the existing views so to take advantage of the built-in Generic Class-Based Views.

There are many other topics that we are going to touch with this tutorial, such as how to work with pagination, how to work with Markdown and how to add a simple editor. We are also going to explore a built-in package called Humanize, which is used to give a "human touch" to the data.

Alright, folks! Let's implement some code. We have plenty of work to do today!

Views Strategies

At the end of the day, all Django views are functions. Even class-based views (CBV). Behind the scenes, it does all the magic and end-up returning a view function.

Class-based views were introduced so to make it easier for developers to reuse and extend views. There are many benefits of using them, such as the extendability, the ability to use O.O. techniques such as multiple inheritances, the handling of HTTP methods are done in separate methods, rather than using conditional branching, and there are also the Generic Class-Based Views (GCBV).

Before we move forward, let's clarify what those three terms mean:

• Function-Based Views (FBV)
• Class-Based Views (CBV)
• Generic Class-Based Views (GCBV)

A FBV is the simplest representation of a Django view: it's just a function that receives an HttpRequest object and returns an HttpResponse.

A CBV is every Django view defined as a Python class that extends the django.views.generic.View abstract class. A CBV essentially is a class that wraps a FBV. CBVs are great to extend and reuse code.

GCBVs are built-in CBVs that solve specific problems such as listing views, create, update, and delete views.

Below we are going to explore some examples of the different implementation strategies.

Function-Based View

views.py

def new_post(request):
    if request.method == 'POST':
        form = PostForm(request.POST)
        if form.is_valid():
            form.save()
            return redirect('post_list')
    else:
        form = PostForm()
    return render(request, 'new_post.html', {'form': form})

urls.py

urlpatterns = [
    url(r'^new_post/$', views.new_post, name='new_post'),
]

Class-Based View

A CBV is a view that extends the View class. The main difference here is that the requests are handled inside class methods named after the HTTP methods, such as get, post, put, head, etc.

So, here we don't need to do a conditional to check if the request is a POST or if it's a GET. The code goes straight to the right method. This logic is handled internally in the View class.

views.py

from django.views.generic import View

class NewPostView(View):
    def post(self, request):
        form = PostForm(request.POST)
        if form.is_valid():
            form.save()
            return redirect('post_list')
        return render(request, 'new_post.html', {'form': form})

    def get(self, request):
        form = PostForm()
        return render(request, 'new_post.html', {'form': form})

The way we refer to the CBVs in the urls.py module is a little bit different:

urls.py

urlpatterns = [
    url(r'^new_post/$', views.NewPostView.as_view(), name='new_post'),
]

Here we need to use the as_view() class method, which returns a view function to the url patterns. In some cases, we can also feed the as_view() with some keyword arguments, so to customize the behavior of the CBV, just like we did with some of the authentication views to customize the templates.

Anyway, the good thing about CBV is that we can add more methods, and perhaps do something like this:

from django.views.generic import View

class NewPostView(View):
    def render(self, request):
        return render(request, 'new_post.html', {'form': self.form})

    def post(self, request):
        self.form = PostForm(request.POST)
        if self.form.is_valid():
            self.form.save()
            return redirect('post_list')
        return self.render(request)

    def get(self, request):
        self.form = PostForm()
        return self.render(request)

It's also possible to create some generic views that accomplish some tasks so that we can reuse it across the project.

But that's pretty much all you need to know about CBVs. Simple as that.

Generic Class-Based View

Now about the GCBV. That's a different story. As I mentioned earlier, those views are built-in CBVs for common use cases. Their implementation makes heavy usage of multiple inheritances (mixins) and other O.O. strategies.

They are very flexible and can save many hours of work. But in the beginning, it can be difficult to work with them.

When I first started working with Django, I found GCBV hard to work with. At first, it's hard to tell what is going on, because the code flow is not obvious, as there is good chunk of code hidden in the parent classes. The documentation is a little bit challenging to follow too, mostly because the attributes and methods are sometimes spread across eight parent classes. When working with GCBV, it's always good to have the ccbv.co.uk opened for quick reference. No worries, we are going to explore it together.

Now let's see a GCBV example.

views.py

from django.views.generic import CreateView

class NewPostView(CreateView):
    model = Post
    form_class = PostForm
    success_url = reverse_lazy('post_list')
    template_name = 'new_post.html'

Here we are using a generic view used to create model objects. It does all the form processing and save the object if the form is valid.

Since it's a CBV, we refer to it in the urls.py the same way as any other CBV:

urls.py

urlpatterns = [
    url(r'^new_post/$', views.NewPostView.as_view(), name='new_post'),
]

Other examples of GCBVs are DetailView, DeleteView, FormView, UpdateView, ListView.

Update View

Let's get back to the implementation of our project. This time we are going to use a GCBV to implement the edit post view:

boards/views.py (view complete file contents)

from django.shortcuts import redirect
from django.views.generic import UpdateView
from django.utils import timezone

class PostUpdateView(UpdateView):
    model = Post
    fields = ('message', )
    template_name = 'edit_post.html'
    pk_url_kwarg = 'topic_pk'
    context_object_name = 'post'

    def form_valid(self, form):
        post = form.save(commit=False)
        post.updated_by = self.request.user
        post.updated_at = timezone.now()
        post.save()
        return redirect('topic_posts', pk=post.topic.board.pk, topic_pk=post.topic.pk)

With the UpdateView and the CreateView, we have the option to either define form_class or the fields attribute. In the example above we are using the fields attribute to create a model form on-the-fly. Internally, Django will use a model form factory to compose a form of the Post model. Since it's a very simple form with just the message field, we can afford to work like this. But for complex form definitions, it's better to define a model form externally and refer to it here.

The pk_url_kwarg will be used to identify the name of the keyword argument used to retrieve the Post object. It's the same as we define in the urls.py.

If we don't set the context_object_name attribute, the Post object will be available in the template as "object." So, here we are using the context_object_name to rename it to post instead. You will see how we are using it in the template below.

In this particular example, we had to override the form_valid() method so to set some extra fields such as the updated_by and updated_at. You can see how the base form_valid() method looks like here: UpdateView#form_valid.

myproject/urls.py (view complete file contents)

from django.conf.urls import url
from boards import views

urlpatterns = [
    # ...
    url(r'^boards/(?P<pk>\d+)/topics/(?P<topic_pk>\d+)/posts/(?P<post_pk>\d+)/edit/$',
        views.PostUpdateView.as_view(), name='edit_post'),
]

Now we can add the link to the edit page:

templates/topic_posts.html (view complete file contents)

{% if post.created_by == user %}
  <div class="mt-3">
    <a href="{% url 'edit_post' post.topic.board.pk post.topic.pk post.pk %}"
       class="btn btn-primary btn-sm"
       role="button">Edit</a>
  </div>
{% endif %}

templates/edit_post.html (view complete file contents)

{% extends 'base.html' %}

{% block title %}Edit post{% endblock %}

{% block breadcrumb %}
  <li class="breadcrumb-item"><a href="{% url 'home' %}">Boards</a></li>
  <li class="breadcrumb-item"><a href="{% url 'board_topics' post.topic.board.pk %}">{{ post.topic.board.name }}</a></li>
  <li class="breadcrumb-item"><a href="{% url 'topic_posts' post.topic.board.pk post.topic.pk %}">{{ post.topic.subject }}</a></li>
  <li class="breadcrumb-item active">Edit post</li>
{% endblock %}

{% block content %}
  <form method="post" class="mb-4" novalidate>
    {% csrf_token %}
    {% include 'includes/form.html' %}
    <button type="submit" class="btn btn-success">Save changes</button>
    <a href="{% url 'topic_posts' post.topic.board.pk post.topic.pk %}" class="btn btn-outline-secondary" role="button">Cancel</a>
  </form>
{% endblock %}

Observe now how we are navigating through the post object: post.topic.board.pk. If we didn't the context_object_name to post, it would be available as: object.topic.board.pk. Got it?

Testing The Update View

Create a new test file named test_view_edit_post.py inside the boards/tests folder. Clicking on the link below you will see many routine tests, just like we have been doing in this tutorial. So I will just highlight the new parts here:

boards/tests/test_view_edit_post.py (view complete file contents)

from django.contrib.auth.models import User
from django.test import TestCase
from django.urls import reverse
from ..models import Board, Post, Topic
from ..views import PostUpdateView

class PostUpdateViewTestCase(TestCase):
    '''
    Base test case to be used in all `PostUpdateView` view tests
    '''
    def setUp(self):
        self.board = Board.objects.create(name='Django', description='Django board.')
        self.username = 'john'
        self.password = '123'
        user = User.objects.create_user(username=self.username, email='john@doe.com', password=self.password)
        self.topic = Topic.objects.create(subject='Hello, world', board=self.board, starter=user)
        self.post = Post.objects.create(message='Lorem ipsum dolor sit amet', topic=self.topic, created_by=user)
        self.url = reverse('edit_post', kwargs={
            'pk': self.board.pk,
            'topic_pk': self.topic.pk,
            'post_pk': self.post.pk
        })

class LoginRequiredPostUpdateViewTests(PostUpdateViewTestCase):
    def test_redirection(self):
        '''
        Test if only logged in users can edit the posts
        '''
        login_url = reverse('login')
        response = self.client.get(self.url)
        self.assertRedirects(response, '{login_url}?next={url}'.format(login_url=login_url, url=self.url))

class UnauthorizedPostUpdateViewTests(PostUpdateViewTestCase):
    def setUp(self):
        '''
        Create a new user different from the one who posted
        '''
        super().setUp()
        username = 'jane'
        password = '321'
        user = User.objects.create_user(username=username, email='jane@doe.com', password=password)
        self.client.login(username=username, password=password)
        self.response = self.client.get(self.url)

    def test_status_code(self):
        '''
        A topic should be edited only by the owner.
        Unauthorized users should get a 404 response (Page Not Found)
        '''
        self.assertEquals(self.response.status_code, 404)


class PostUpdateViewTests(PostUpdateViewTestCase):
    # ...

class SuccessfulPostUpdateViewTests(PostUpdateViewTestCase):
    # ...

class InvalidPostUpdateViewTests(PostUpdateViewTestCase):
    # ...

For now, the important parts are: PostUpdateViewTestCase is a class we defined to be reused across the other test cases. It's just the basic setup, creating users, topic, boards, and so on.

The class LoginRequiredPostUpdateViewTests will test if the view is protected with the @login_required decorator. That is if only authenticated users can access the edit page.

The class UnauthorizedPostUpdateViewTests creates a new user, different from the one who posted and tries to access the edit page. The application should only authorize the owner of the post to edit it.

Let's run the tests:

python manage.py test boards.tests.test_view_edit_post

Creating test database for alias 'default'...
System check identified no issues (0 silenced).
..F.......F
======================================================================
FAIL: test_redirection (boards.tests.test_view_edit_post.LoginRequiredPostUpdateViewTests)
----------------------------------------------------------------------
...
AssertionError: 200 != 302 : Response didn't redirect as expected: Response code was 200 (expected 302)

======================================================================
FAIL: test_status_code (boards.tests.test_view_edit_post.UnauthorizedPostUpdateViewTests)
----------------------------------------------------------------------
...
AssertionError: 200 != 404

----------------------------------------------------------------------
Ran 11 tests in 1.360s

FAILED (failures=2)
Destroying test database for alias 'default'...

First, let's fix the problem with the @login_required decorator. The way we use view decorators on class-based views is a little bit different. We need an extra import:

boards/views.py (view complete file contents)

from django.contrib.auth.decorators import login_required
from django.shortcuts import redirect
from django.views.generic import UpdateView
from django.utils import timezone
from django.utils.decorators import method_decorator
from .models import Post

@method_decorator(login_required, name='dispatch')
class PostUpdateView(UpdateView):
    model = Post
    fields = ('message', )
    template_name = 'edit_post.html'
    pk_url_kwarg = 'post_pk'
    context_object_name = 'post'

    def form_valid(self, form):
        post = form.save(commit=False)
        post.updated_by = self.request.user
        post.updated_at = timezone.now()
        post.save()
        return redirect('topic_posts', pk=post.topic.board.pk, topic_pk=post.topic.pk)

We can't decorate the class directly with the @login_required decorator. We have to use the utility @method_decorator, and pass a decorator (or a list of decorators) and tell which method should be decorated. In class-based views it's common to decorate the dispatch method. It's an internal method Django use (defined inside the View class). All requests pass through this method, so it's safe to decorate it.

Run the tests:

python manage.py test boards.tests.test_view_edit_post

Creating test database for alias 'default'...
System check identified no issues (0 silenced).
..........F
======================================================================
FAIL: test_status_code (boards.tests.test_view_edit_post.UnauthorizedPostUpdateViewTests)
----------------------------------------------------------------------
...
AssertionError: 200 != 404

----------------------------------------------------------------------
Ran 11 tests in 1.353s

FAILED (failures=1)
Destroying test database for alias 'default'...

Okay! We fixed the @login_required problem. Now we have to deal with the other users editing any posts problem.

The easiest way to solve this problem is by overriding the get_queryset method of the UpdateView. You can see what the original method looks like here UpdateView#get_queryset.

boards/views.py (view complete file contents)

@method_decorator(login_required, name='dispatch')
class PostUpdateView(UpdateView):
    model = Post
    fields = ('message', )
    template_name = 'edit_post.html'
    pk_url_kwarg = 'post_pk'
    context_object_name = 'post'

    def get_queryset(self):
        queryset = super().get_queryset()
        return queryset.filter(created_by=self.request.user)

    def form_valid(self, form):
        post = form.save(commit=False)
        post.updated_by = self.request.user
        post.updated_at = timezone.now()
        post.save()
        return redirect('topic_posts', pk=post.topic.board.pk, topic_pk=post.topic.pk)

With the line queryset = super().get_queryset() we are reusing the get_queryset method from the parent class, that is, the UpateView class. Then, we are adding an extra filter to the queryset, which is filtering the post using the logged in user, available inside the request object.

Test it again:

python manage.py test boards.tests.test_view_edit_post

Creating test database for alias 'default'...
System check identified no issues (0 silenced).
...........
----------------------------------------------------------------------
Ran 11 tests in 1.321s

OK
Destroying test database for alias 'default'...

All good!

List View

We could refactor some of our existing views to take advantage of the CBV capabilities. Take the home page for example. We are just grabbing all the boards from the database and listing it in the HTML:

boards/views.py

from django.shortcuts import render
from .models import Board

def home(request):
    boards = Board.objects.all()
    return render(request, 'home.html', {'boards': boards})

Here is how we could rewrite it using a GCBV for models listing:

boards/views.py (view complete file contents)

from django.views.generic import ListView
from .models import Board

class BoardListView(ListView):
    model = Board
    context_object_name = 'boards'
    template_name = 'home.html'

Then we have to change the reference in the urls.py module:

myproject/urls.py (view complete file contents)

from django.conf.urls import url
from boards import views

urlpatterns = [
    url(r'^$', views.BoardListView.as_view(), name='home'),
    # ...
]

If we check the homepage we will see that nothing really changed, everything is working as expected. But we have to tweak our tests a little bit because now we are dealing with a class-based view:

boards/tests/test_view_home.py (view complete file contents)

from django.test import TestCase
from django.urls import resolve
from ..views import BoardListView

class HomeTests(TestCase):
    # ...
    def test_home_url_resolves_home_view(self):
        view = resolve('/')
        self.assertEquals(view.func.view_class, BoardListView)

Pagination

We can very easily implement pagination with class-based views. But first I wanted to do a pagination by hand, so that we can explore better the mechanics behind it, so it doesn't look like magic.

It wouldn't really make sense to paginate the boards listing view because we do not expect to have many boards. But definitively the topics listing and the posts listing needs some pagination.

From now on, we will be working on the board_topics view.

First, let's add some volume of posts. We could just use the application's user interface and add several posts, or open the Python shell and write a small script to do it for us:

python manage.py shell

from django.contrib.auth.models import User
from boards.models import Board, Topic, Post

user = User.objects.first()

board = Board.objects.get(name='Django')

for i in range(100):
    subject = 'Topic test #{}'.format(i)
    topic = Topic.objects.create(subject=subject, board=board, starter=user)
    Post.objects.create(message='Lorem ipsum...', topic=topic, created_by=user)

Good, now we have some data to play with.

Before we jump into the code, let's experiment a little bit more with the Python shell:

python manage.py shell

from boards.models import Topic

# All the topics in the app
Topic.objects.count()
107

# Just the topics in the Django board
Topic.objects.filter(board__name='Django').count()
104

# Let's save this queryset into a variable to paginate it
queryset = Topic.objects.filter(board__name='Django').order_by('-last_updated')

It's very important always define an ordering to a QuerySet you are going to paginate! Otherwise, it can give you inconsistent results.

Now let's import the Paginator utility:

from django.core.paginator import Paginator

paginator = Paginator(queryset, 20)

Here we are telling Django to paginate our QuerySet in pages of 20 each. Now let's explore some of the paginator properties:

# count the number of elements in the paginator
paginator.count
104

# total number of pages
# 104 elements, paginating 20 per page gives you 6 pages
# where the last page will have only 4 elements
paginator.num_pages
6

# range of pages that can be used to iterate and create the
# links to the pages in the template
paginator.page_range
range(1, 7)

# returns a Page instance
paginator.page(2)
<Page 2 of 6>

page = paginator.page(2)

type(page)
django.core.paginator.Page

type(paginator)
django.core.paginator.Paginator

Here we have to pay attention because if we try to get a page that doesn't exist, the Paginator will throw an exception:

paginator.page(7)
EmptyPage: That page contains no results

Or if we try to pass an arbitrary parameter, which is not a page number:

paginator.page('abc')
PageNotAnInteger: That page number is not an integer

We have to keep those details in mind when designing the user interface.

Now let's explore a little bit the attributes and methods offered by the Page class:

page = paginator.page(1)

# Check if there is another page after this one
page.has_next()
True

# If there is no previous page, that means this one is the first page
page.has_previous()
False

page.has_other_pages()
True

page.next_page_number()
2

# Take care here, since there is no previous page,
# if we call the method `previous_page_number() we will get an exception:
page.previous_page_number()
EmptyPage: That page number is less than 1

FBV Pagination

Here is how we do it using regular function-based views:

boards/views.py (view complete file contents)

from django.db.models import Count
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.shortcuts import get_object_or_404, render
from django.views.generic import ListView
from .models import Board

def board_topics(request, pk):
    board = get_object_or_404(Board, pk=pk)
    queryset = board.topics.order_by('-last_updated').annotate(replies=Count('posts') - 1)
    page = request.GET.get('page', 1)

    paginator = Paginator(queryset, 20)

    try:
        topics = paginator.page(page)
    except PageNotAnInteger:
        # fallback to the first page
        topics = paginator.page(1)
    except EmptyPage:
        # probably the user tried to add a page number
        # in the url, so we fallback to the last page
        topics = paginator.page(paginator.num_pages)

    return render(request, 'topics.html', {'board': board, 'topics': topics})

Now the trick part is to render the pages correctly using the Bootstrap 4 pagination component. But take the time to read the code and see if it makes sense for you. We are using here all the methods we played with before. And here in that context, topics is no longer a QuerySet but a paginator.Page instance.

Right after the topics HTML table, we can render the pagination component:

templates/topics.html (view complete file contents)

{% if topics.has_other_pages %}
  <nav aria-label="Topics pagination" class="mb-4">
    <ul class="pagination">
      {% if topics.has_previous %}
        <li class="page-item">
          <a class="page-link" href="?page={{ topics.previous_page_number }}">Previous</a>
        </li>
      {% else %}
        <li class="page-item disabled">
          <span class="page-link">Previous</span>
        </li>
      {% endif %}

      {% for page_num in topics.paginator.page_range %}
        {% if topics.number == page_num %}
          <li class="page-item active">
            <span class="page-link">
              {{ page_num }}
              <span class="sr-only">(current)</span>
            </span>
          </li>
        {% else %}
          <li class="page-item">
            <a class="page-link" href="?page={{ page_num }}">{{ page_num }}</a>
          </li>
        {% endif %}
      {% endfor %}

      {% if topics.has_next %}
        <li class="page-item">
          <a class="page-link" href="?page={{ topics.next_page_number }}">Next</a>
        </li>
      {% else %}
        <li class="page-item disabled">
          <span class="page-link">Next</span>
        </li>
      {% endif %}
    </ul>
  </nav>
{% endif %}

GCBV Pagination

Below, the same implementation but this time using the ListView.

boards/views.py (view complete file contents)

class TopicListView(ListView):
    model = Topic
    context_object_name = 'topics'
    template_name = 'topics.html'
    paginate_by = 20

    def get_context_data(self, **kwargs):
        kwargs['board'] = self.board
        return super().get_context_data(**kwargs)

    def get_queryset(self):
        self.board = get_object_or_404(Board, pk=self.kwargs.get('pk'))
        queryset = self.board.topics.order_by('-last_updated').annotate(replies=Count('posts') - 1)
        return queryset

While using pagination with class-based views, the way we interact with the paginator in the template is a little bit different. It will make available the following variables in the template: paginator, page_obj, is_paginated, object_list, and also a variable with the name we defined in the context_object_name. In our case this extra variable will be named topics, and it will be equivalent to object_list.

Now about the whole get_context_data thing, well, that's how we add stuff to the request context when extending a GCBV.

But the main point here is the paginate_by attribute. In some cases, just by adding it will be enough.

Remember to update the urls.py:

myproject/urls.py (view complete file contents)

from django.conf.urls import url
from boards import views

urlpatterns = [
    # ...
    url(r'^boards/(?P<pk>\d+)/$', views.TopicListView.as_view(), name='board_topics'),
]

Now let's fix the template:

templates/topics.html (view complete file contents)

{% block content %}
  <div class="mb-4">
    <a href="{% url 'new_topic' board.pk %}" class="btn btn-primary">New topic</a>
  </div>

  <table class="table mb-4">
    <!-- table content suppressed -->
  </table>

  {% if is_paginated %}
    <nav aria-label="Topics pagination" class="mb-4">
      <ul class="pagination">
        {% if page_obj.has_previous %}
          <li class="page-item">
            <a class="page-link" href="?page={{ page_obj.previous_page_number }}">Previous</a>
          </li>
        {% else %}
          <li class="page-item disabled">
            <span class="page-link">Previous</span>
          </li>
        {% endif %}

        {% for page_num in paginator.page_range %}
          {% if page_obj.number == page_num %}
            <li class="page-item active">
              <span class="page-link">
                {{ page_num }}
                <span class="sr-only">(current)</span>
              </span>
            </li>
          {% else %}
            <li class="page-item">
              <a class="page-link" href="?page={{ page_num }}">{{ page_num }}</a>
            </li>
          {% endif %}
        {% endfor %}

        {% if page_obj.has_next %}
          <li class="page-item">
            <a class="page-link" href="?page={{ page_obj.next_page_number }}">Next</a>
          </li>
        {% else %}
          <li class="page-item disabled">
            <span class="page-link">Next</span>
          </li>
        {% endif %}
      </ul>
    </nav>
  {% endif %}

{% endblock %}

Now take the time to run the tests and fix if needed.

boards/tests/test_view_board_topics.py

from django.test import TestCase
from django.urls import resolve
from ..views import TopicListView

class BoardTopicsTests(TestCase):
    # ...
    def test_board_topics_url_resolves_board_topics_view(self):
        view = resolve('/boards/1/')
        self.assertEquals(view.func.view_class, TopicListView)

Reusable Pagination Template

Just like we did with the form.html partial template, we can also create something similar for the pagination HTML snippet.

Let's paginate the topic posts page, and then find a way to reuse the pagination component.

boards/views.py (view complete file contents)

class PostListView(ListView):
    model = Post
    context_object_name = 'posts'
    template_name = 'topic_posts.html'
    paginate_by = 2

    def get_context_data(self, **kwargs):
        self.topic.views += 1
        self.topic.save()
        kwargs['topic'] = self.topic
        return super().get_context_data(**kwargs)

    def get_queryset(self):
        self.topic = get_object_or_404(Topic, board__pk=self.kwargs.get('pk'), pk=self.kwargs.get('topic_pk'))
        queryset = self.topic.posts.order_by('created_at')
        return queryset

Now update the urls.py (view complete file contents)

from django.conf.urls import url
from boards import views

urlpatterns = [
    # ...
    url(r'^boards/(?P<pk>\d+)/topics/(?P<topic_pk>\d+)/$', views.PostListView.as_view(), name='topic_posts'),
]

Now we grab that pagination HTML snippet from the topics.html template, and create a new file named pagination.html inside the templates/includes folder, alongside with the forms.html file:

myproject/
 |-- myproject/
 |    |-- accounts/
 |    |-- boards/
 |    |-- myproject/
 |    |-- static/
 |    |-- templates/
 |    |    |-- includes/
 |    |    |    |-- form.html
 |    |    |    +-- pagination.html  <-- here!
 |    |    +-- ...
 |    |-- db.sqlite3
 |    +-- manage.py
 +-- venv/

templates/includes/pagination.html

{% if is_paginated %}
  <nav aria-label="Topics pagination" class="mb-4">
    <ul class="pagination">
      {% if page_obj.has_previous %}
        <li class="page-item">
          <a class="page-link" href="?page={{ page_obj.previous_page_number }}">Previous</a>
        </li>
      {% else %}
        <li class="page-item disabled">
          <span class="page-link">Previous</span>
        </li>
      {% endif %}

      {% for page_num in paginator.page_range %}
        {% if page_obj.number == page_num %}
          <li class="page-item active">
            <span class="page-link">
              {{ page_num }}
              <span class="sr-only">(current)</span>
            </span>
          </li>
        {% else %}
          <li class="page-item">
            <a class="page-link" href="?page={{ page_num }}">{{ page_num }}</a>
          </li>
        {% endif %}
      {% endfor %}

      {% if page_obj.has_next %}
        <li class="page-item">
          <a class="page-link" href="?page={{ page_obj.next_page_number }}">Next</a>
        </li>
      {% else %}
        <li class="page-item disabled">
          <span class="page-link">Next</span>
        </li>
      {% endif %}
    </ul>
  </nav>
{% endif %}

Now in the topic_posts.html template we use it:

templates/topic_posts.html (view complete file contents)

{% block content %}

  <div class="mb-4">
    <a href="{% url 'reply_topic' topic.board.pk topic.pk %}" class="btn btn-primary" role="button">Reply</a>
  </div>

  {% for post in posts %}
    <div class="card {% if forloop.last %}mb-4{% else %}mb-2{% endif %} {% if forloop.first %}border-dark{% endif %}">
      {% if forloop.first %}
        <div class="card-header text-white bg-dark py-2 px-3">{{ topic.subject }}</div>
      {% endif %}
      <div class="card-body p-3">
        <div class="row">
          <div class="col-2">
            <img src="{% static 'img/avatar.svg' %}" alt="{{ post.created_by.username }}" class="w-100">
            <small>Posts: {{ post.created_by.posts.count }}</small>
          </div>
          <div class="col-10">
            <div class="row mb-3">
              <div class="col-6">
                <strong class="text-muted">{{ post.created_by.username }}</strong>
              </div>
              <div class="col-6 text-right">
                <small class="text-muted">{{ post.created_at }}</small>
              </div>
            </div>
            {{ post.message }}
            {% if post.created_by == user %}
              <div class="mt-3">
                <a href="{% url 'edit_post' post.topic.board.pk post.topic.pk post.pk %}"
                   class="btn btn-primary btn-sm"
                   role="button">Edit</a>
              </div>
            {% endif %}
          </div>
        </div>
      </div>
    </div>
  {% endfor %}

  {% include 'includes/pagination.html' %}

{% endblock %}

Don't forget to change the main forloop to {% for post in posts %}.

We can also update the previous template, the topics.html template to use the pagination partial template too:

templates/topics.html (view complete file contents)

{% block content %}
  <div class="mb-4">
    <a href="{% url 'new_topic' board.pk %}" class="btn btn-primary">New topic</a>
  </div>

  <table class="table mb-4">
    <!-- table code suppressed -->
  </table>

  {% include 'includes/pagination.html' %}

{% endblock %}

Just for testing purpose, you could just add a few posts (or create some using the Python Shell) and change the paginate_by to a low number, say 2, and see how it's looking like:

(view complete file contents)

Update the test cases:

boards/tests/test_view_topic_posts.py

from django.test import TestCase
from django.urls import resolve
from ..views import PostListView

class TopicPostsTests(TestCase):
    # ...
    def test_view_function(self):
        view = resolve('/boards/1/topics/1/')
        self.assertEquals(view.func.view_class, PostListView)

My Account View

Okay, so, this is going to be our last view. After that, we will be working on improving the existing features.

accounts/views.py (view complete file contents)

from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from django.urls import reverse_lazy
from django.utils.decorators import method_decorator
from django.views.generic import UpdateView

@method_decorator(login_required, name='dispatch')
class UserUpdateView(UpdateView):
    model = User
    fields = ('first_name', 'last_name', 'email', )
    template_name = 'my_account.html'
    success_url = reverse_lazy('my_account')

    def get_object(self):
        return self.request.user

myproject/urls.py (view complete file contents)

from django.conf.urls import url
from accounts import views as accounts_views

urlpatterns = [
    # ...
    url(r'^settings/account/$', accounts_views.UserUpdateView.as_view(), name='my_account'),
]

templates/my_account.html

{% extends 'base.html' %}

{% block title %}My account{% endblock %}

{% block breadcrumb %}
  <li class="breadcrumb-item active">My account</li>
{% endblock %}

{% block content %}
  <div class="row">
    <div class="col-lg-6 col-md-8 col-sm-10">
      <form method="post" novalidate>
        {% csrf_token %}
        {% include 'includes/form.html' %}
        <button type="submit" class="btn btn-success">Save changes</button>
      </form>
    </div>
  </div>
{% endblock %}

Adding Markdown

Let's improve the user experience by adding Markdown to our text areas. You will see it's very easy and simple.

First, let's install a library called Python-Markdown:

pip install markdown

We can add a new method to the Post model:

boards/models.py (view complete file contents)

from django.db import models
from django.utils.html import mark_safe
from markdown import markdown

class Post(models.Model):
    # ...

    def get_message_as_markdown(self):
        return mark_safe(markdown(self.message, safe_mode='escape'))

Here we are dealing with user input, so we must take care. When using the markdown function, we are instructing it to escape the special characters first and then parse the markdown tags. After that, we mark the output string as safe to be used in the template.

Now in the templates topic_posts.html and reply_topic.html just change from:

{{ post.message }}

To:

{{ post.get_message_as_markdown }}

From now on the users can already use markdown in the posts:

Markdown Editor

We can also add a very cool Markdown editor called SimpleMD.

Either download the JavaScript library or use their CDN:

<link rel="stylesheet" href="https://cdn.jsdelivr.net/simplemde/latest/simplemde.min.css">
<script src="https://cdn.jsdelivr.net/simplemde/latest/simplemde.min.js"></script>

Now edit the base.html to make space for extra JavaScripts:

templates/base.html (view complete file contents)

    <script src="{% static 'js/jquery-3.2.1.min.js' %}"></script>
    <script src="{% static 'js/popper.min.js' %}"></script>
    <script src="{% static 'js/bootstrap.min.js' %}"></script>
    {% block javascript %}{% endblock %}  <!-- Add this empty block here! -->

First edit the reply_topic.html template:

templates/reply_topic.html (view complete file contents)

{% extends 'base.html' %}

{% load static %}

{% block title %}Post a reply{% endblock %}

{% block stylesheet %}
  <link rel="stylesheet" href="{% static 'css/simplemde.min.css' %}">
{% endblock %}

{% block javascript %}
  <script src="{% static 'js/simplemde.min.js' %}"></script>
  <script>
    var simplemde = new SimpleMDE();
  </script>
{% endblock %}

By default, this plugin will transform the first text area it finds into a markdown editor. So just that code should be enough:

Now do the same thing with the edit_post.html template:

templates/edit_post.html (view complete file contents)

{% extends 'base.html' %}

{% load static %}

{% block title %}Edit post{% endblock %}

{% block stylesheet %}
  <link rel="stylesheet" href="{% static 'css/simplemde.min.css' %}">
{% endblock %}

{% block javascript %}
  <script src="{% static 'js/simplemde.min.js' %}"></script>
  <script>
    var simplemde = new SimpleMDE();
  </script>
{% endblock %}

Humanize

I just thought it would be a nice touch to add the built-in humanize package. It's a set of utility functions to add a "human touch" to data.

For example, we can use it to display date and time fields more naturally. Instead of showing the whole date, we can simply show: "2 minutes ago".

Let's do it. First, add the django.contrib.humanize to the INSTALLED_APPS.

myproject/settings.py

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.humanize',  # <- here

    'widget_tweaks',

    'accounts',
    'boards',
]

Now we can use it in the templates. First, let's edit the topics.html template:

templates/topics.html (view complete file contents)

{% extends 'base.html' %}

{% load humanize %}

{% block content %}
  <!-- code suppressed -->

  <td>{{ topic.last_updated|naturaltime }}</td>

  <!-- code suppressed -->
{% endblock %}

All we have to do is to load the template tags using {% load humanize %} and then apply the template filter: {{ topic.last_updated|naturaltime }}

You may add it to other places you like.

Gravatar

A very easy way to add user profile pictures is by using Gravatar.

Inside the boards/templatetags folder, create a new file named gravatar.py:

boards/templatetags/gravatar.py

import hashlib
from urllib.parse import urlencode

from django import template
from django.conf import settings

register = template.Library()


@register.filter
def gravatar(user):
    email = user.email.lower().encode('utf-8')
    default = 'mm'
    size = 256
    url = 'https://www.gravatar.com/avatar/{md5}?{params}'.format(
        md5=hashlib.md5(email).hexdigest(),
        params=urlencode({'d': default, 's': str(size)})
    )
    return url

Basically I'm using the code snippet they provide. I just adapted it to work with Python 3.

Great, now we can load it in our template, just like we did with the Humanize template filter:

templates/topic_posts.html (view complete file contents)

{% extends 'base.html' %}

{% load gravatar %}

{% block content %}
  <!-- code suppressed -->

  <img src="{{ post.created_by|gravatar }}" alt="{{ post.created_by.username }}" class="w-100 rounded">

  <!-- code suppressed -->
{% endblock %}

Final Adjustments

Maybe you have already noticed, but there's a small issue when someone replies to a post. It's not updating the

boards/views.py

@login_required
def reply_topic(request, pk, topic_pk):
    topic = get_object_or_404(Topic, board__pk=pk, pk=topic_pk)
    if request.method == 'POST':
        form = PostForm(request.POST)
        if form.is_valid():
            post = form.save(commit=False)
            post.topic = topic
            post.created_by = request.user
            post.save()

            topic.last_updated = timezone.now()  # <- here
            topic.save()                         # <- and here

            return redirect('topic_posts', pk=pk, topic_pk=topic_pk)
    else:
        form = PostForm()
    return render(request, 'reply_topic.html', {'topic': topic, 'form': form})

Next thing we want to do is try to control a little bit the view counting system. We don't want to the same user refreshing the page counting multiple views. For this we can use sessions:

boards/views.py

class PostListView(ListView):
    model = Post
    context_object_name = 'posts'
    template_name = 'topic_posts.html'
    paginate_by = 20

    def get_context_data(self, **kwargs):

        session_key = 'viewed_topic_{}'.format(self.topic.pk)  # <-- here
        if not self.request.session.get(session_key, False):
            self.topic.views += 1
            self.topic.save()
            self.request.session[session_key] = True           # <-- until here

        kwargs['topic'] = self.topic
        return super().get_context_data(**kwargs)

    def get_queryset(self):
        self.topic = get_object_or_404(Topic, board__pk=self.kwargs.get('pk'), pk=self.kwargs.get('topic_pk'))
        queryset = self.topic.posts.order_by('created_at')
        return queryset

Now we could provide a better navigation in the topics listing. Currently the only option is for the user to click in the topic title and go to the first page. We could workout something like this:

boards/models.py

import math
from django.db import models

class Topic(models.Model):
    # ...

    def __str__(self):
        return self.subject

    def get_page_count(self):
        count = self.posts.count()
        pages = count / 20
        return math.ceil(pages)

    def has_many_pages(self, count=None):
        if count is None:
            count = self.get_page_count()
        return count > 6

    def get_page_range(self):
        count = self.get_page_count()
        if self.has_many_pages(count):
            return range(1, 5)
        return range(1, count + 1)

Then in the topics.html template we could implement something like this:

templates/topics.html

  <table class="table table-striped mb-4">
    <thead class="thead-inverse">
      <tr>
        <th>Topic</th>
        <th>Starter</th>
        <th>Replies</th>
        <th>Views</th>
        <th>Last Update</th>
      </tr>
    </thead>
    <tbody>
      {% for topic in topics %}
        {% url 'topic_posts' board.pk topic.pk as topic_url %}
        <tr>
          <td>
            <p class="mb-0">
              <a href="{{ topic_url }}">{{ topic.subject }}</a>
            </p>
            <small class="text-muted">
              Pages:
              {% for i in topic.get_page_range %}
                <a href="{{ topic_url }}?page={{ i }}">{{ i }}</a>
              {% endfor %}
              {% if topic.has_many_pages %}
              ... <a href="{{ topic_url }}?page={{ topic.get_page_count }}">Last Page</a>
              {% endif %}
            </small>
          </td>
          <td class="align-middle">{{ topic.starter.username }}</td>
          <td class="align-middle">{{ topic.replies }}</td>
          <td class="align-middle">{{ topic.views }}</td>
          <td class="align-middle">{{ topic.last_updated|naturaltime }}</td>
        </tr>
      {% endfor %}
    </tbody>
  </table>

Like a tiny pagination for each topic. Note that I also took the time to add the table-striped class for a better styling of the table.

In the reply page, we are currently listing all topic replies. We could limit it to just the last ten posts.

boards/models.py

class Topic(models.Model):
    # ...

    def get_last_ten_posts(self):
        return self.posts.order_by('-created_at')[:10]

templates/reply_topic.html

{% block content %}

  <form method="post" class="mb-4" novalidate>
    {% csrf_token %}
    {% include 'includes/form.html' %}
    <button type="submit" class="btn btn-success">Post a reply</button>
  </form>

  {% for post in topic.get_last_ten_posts %}  <!-- here! -->
    <div class="card mb-2">
      <!-- code suppressed -->
    </div>
  {% endfor %}

{% endblock %}

Another thing is that when the user replies to a post, we are redirecting the user to the first page again. We could improve it by sending the user to the last page.

We can add an id to the post card:

templates/topic_posts.html

{% block content %}

  <div class="mb-4">
    <a href="{% url 'reply_topic' topic.board.pk topic.pk %}" class="btn btn-primary" role="button">Reply</a>
  </div>

  {% for post in posts %}
    <div id="{{ post.pk }}" class="card {% if forloop.last %}mb-4{% else %}mb-2{% endif %} {% if forloop.first %}border-dark{% endif %}">
      <!-- code suppressed -->
    </div>
  {% endfor %}

  {% include 'includes/pagination.html' %}

{% endblock %}

The important bit here is the <div id="{{ post.pk }}" ...>.

Then we can play with it like this in the views:

boards/views.py

@login_required
def reply_topic(request, pk, topic_pk):
    topic = get_object_or_404(Topic, board__pk=pk, pk=topic_pk)
    if request.method == 'POST':
        form = PostForm(request.POST)
        if form.is_valid():
            post = form.save(commit=False)
            post.topic = topic
            post.created_by = request.user
            post.save()

            topic.last_updated = timezone.now()
            topic.save()

            topic_url = reverse('topic_posts', kwargs={'pk': pk, 'topic_pk': topic_pk})
            topic_post_url = '{url}?page={page}#{id}'.format(
                url=topic_url,
                id=post.pk,
                page=topic.get_page_count()
            )

            return redirect(topic_post_url)
    else:
        form = PostForm()
    return render(request, 'reply_topic.html', {'topic': topic, 'form': form})

In the topic_post_url we are building a URL with the last page and adding an anchor to the element with id equals to the post ID.

With this, it will required us to update the following test case:

boards/tests/test_view_reply_topic.py

class SuccessfulReplyTopicTests(ReplyTopicTestCase):
    # ...

    def test_redirection(self):
        '''
        A valid form submission should redirect the user
        '''
        url = reverse('topic_posts', kwargs={'pk': self.board.pk, 'topic_pk': self.topic.pk})
        topic_posts_url = '{url}?page=1#2'.format(url=url)
        self.assertRedirects(self.response, topic_posts_url)

Next issue, as you can see in the previous screenshot, is to solve the problem with the pagination when the number of pages is too high.

The easiest way is to tweak the pagination.html template:

templates/includes/pagination.html

{% if is_paginated %}
  <nav aria-label="Topics pagination" class="mb-4">
    <ul class="pagination">
      {% if page_obj.number > 1 %}
        <li class="page-item">
          <a class="page-link" href="?page=1">First</a>
        </li>
      {% else %}
        <li class="page-item disabled">
          <span class="page-link">First</span>
        </li>
      {% endif %}

      {% if page_obj.has_previous %}
        <li class="page-item">
          <a class="page-link" href="?page={{ page_obj.previous_page_number }}">Previous</a>
        </li>
      {% else %}
        <li class="page-item disabled">
          <span class="page-link">Previous</span>
        </li>
      {% endif %}

      {% for page_num in paginator.page_range %}
        {% if page_obj.number == page_num %}
          <li class="page-item active">
            <span class="page-link">
              {{ page_num }}
              <span class="sr-only">(current)</span>
            </span>
          </li>
        {% elif page_num > page_obj.number|add:'-3' and page_num < page_obj.number|add:'3' %}
          <li class="page-item">
            <a class="page-link" href="?page={{ page_num }}">{{ page_num }}</a>
          </li>
        {% endif %}
      {% endfor %}

      {% if page_obj.has_next %}
        <li class="page-item">
          <a class="page-link" href="?page={{ page_obj.next_page_number }}">Next</a>
        </li>
      {% else %}
        <li class="page-item disabled">
          <span class="page-link">Next</span>
        </li>
      {% endif %}

      {% if page_obj.number != paginator.num_pages %}
        <li class="page-item">
          <a class="page-link" href="?page={{ paginator.num_pages }}">Last</a>
        </li>
      {% else %}
        <li class="page-item disabled">
          <span class="page-link">Last</span>
        </li>
      {% endif %}
    </ul>
  </nav>
{% endif %}

Conclusions

With this tutorial, we finalized the implementation of our Django board application. I will probably release a follow-up implementation tutorial to improve the code. There are many things we can explore together. For example, database optimizations, improve the user interface, play with file uploads, create a moderation system, and so on.

The next tutorial will be focused on deployment. It's going to be a complete guide on how to put your code in production taking care of all the important details.

I hope you enjoyed the sixth part of this tutorial series! The last part is coming out next week, on Oct 16, 2017. If you would like to get notified when the last part is out, you can subscribe to our mailing list.

The source code of the project is available on GitHub. The current state of the project can be found under the release tag v0.6-lw. The link below will take you to the right place:

https://github.com/sibtc/django-beginners-guide/tree/v0.6-lw

← Part 5 - Django ORM

Part 7 - Deployment (coming soon)

Subscribe to our mailing list to get notified when it's out.

09 Oct 2017 10:00am GMT

Haystack works as a search plugin for Django. You can use different backends Elastic-search, Whose, Sorl, Xapian to search objects. All backends work with the same code. In this post, I am using elastic search as backend.

Installation:

pip install django-haystack

Configuration:

add haystack to installed apps

    INSTALLED_APPS=[

               'django.contrib.admin',
               'django.contrib.auth',
               'django.contrib.contenttypes',
               'django.contrib.sessions',
               'django.contrib.sites',
               #add haystack here
               'haystack',
               'books'
    ]

Settings:

Add back-end settings for the haystack.

     HAYSTACK_CONNECTIONS = {
              'default': {
                    'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
                    'URL': 'http://127.0.0.1:9200/',
                    'INDEX_NAME': 'haystack_books',
              },
    }

Above settings for elastic search.

Add signal processor for the haystack. This signal will update objects in an index.

    HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'

For This tutorial, I am using below as a model.

books/models.py:

    class Book(models.Model):
        title = models.CharField(max_length=100, unique=True)
        slug = models.SlugField(unique=True)
        created_on = models.DateTimeField(auto_now_add=True)
        description = models.TextField()
        authors = models.ManyToManyField(settings.AUTH_USER_MODEL)
        
        def __unicode__(self):
            return self.title

SearchIndex:

The purpose of indexing objects is to optimize speed and performance in finding relevant documents for a given search query.
With search index haystack determines what data should be placed in the index. In haystack, you write a unique index for each model.

create a new file in books/search_indexes.py below code goes

    from haystack import indexes
    from books.models import Book

    class BookIndex(indexes.SearchIndex, indexes.Indexable):
        text = indexes.CharField(document=True, use_template=True, template_name="search/book_text.txt")
        title = indexes.CharField(model_attr='title')
        authors = indexes.CharField()

        def get_model(self):
            return Book

        def prepare_authors(self, obj):
            return [ author.name for a in obj.authors.all()]

        def index_queryset(self, using=None):
            return self.get_model().objects.all()

search index will be created by subclassing both haystack.Indexes.SearchIndex and haystack.Indexes.Indexable. Specify index field types by indexes field types(similar to models).

in above index class get_model method used to define a model which you want to create indexes. prepare_fieldname method used to define data for indexing.

Haystack will use a template to index data. these index syntaxes are similar to Django template tags.

templates/search/book_text.txt

    {{ object.title }}
    {% for a in authors%}
        {{ a }}
    {% endfor %}

Setup:

urls.py

    urlpatterns = patterns(
        # other urls
        url(r'^search/', include('haystack.urls')),
    )

build index:

Python manage.py rebuild_index

search query form.

<form action="/search" method="get">
        <input name="q" type="text" />
        <input type="submit" />
    </form>

input field q is mandatory for searching.

Display search results:

belove code goes in templates/search/search.html

        {% for result in page.object_list %}
           {{ result.object.title }}
           {{ result.object.description }}
        {{ result.object.created_on }}
        {% empty %}           
             No results found.
        {% endfor %}

Got error or want to give suggestions please comment below. Happy Coding.

09 Oct 2017 5:00am GMT

Django's built-in User model a...

08 Oct 2017 10:28pm GMT

# The Simple Power of Django V...

07 Oct 2017 11:33pm GMT

Permissions:

Actually permissions are of 2 types:

1.Model level permissions
2.object level permissions

If you want to give permissions on all cars, then Model-level is appropriate, but if you Django to give permissions on a per-car basis you want Object-level. You may need both, and this isn't a problem as we'll see.

For Model permissions, Django will create permissions in the form 'appname.permissionname_modelname' for each model. If you have an app called 'drivers' with the Car model then one permission would be 'drivers.delete_car'. The permissions that Django automatically creates will be create, change, and delete.Read permission is not included in CRUD operation.Django decided to change CRUD's 'update' to 'change' for some reason. To add more permissions to a model, say read permissions, you use the Meta class:

class Book( models.Model ):

    # model stuff here

    class Meta:

        permissions = ( 

            ( "read_book", "Can read book" ),

        )

Permissions is a set of tuples, where the tuple items are the permission as described above and a description of that permission.

Finally, to check permissions, you can use has_perm:

obj.has_perm( 'drivers.read_car' )

Where obj is either a User or Group instance.

Here is some example which is used to check permissions 'perms' of a model object called entity in the app

def has_model_permissions( entity, model, perms, app ):

    for p in perms:

        if not entity.has_perm( "%s.%s_%s" % ( app, p, model.__name__ ) ):

            return False

        return True

Where entity is the object to check permissions on (Group or User), model is the instance of a model, perms is a list of permissions as strings to check (e.g. ['read', 'change']), and app is the application name as a string. To do the same check as has_perm above you'd call something like this:

result = has_model_permissions( myuser, mycar, ['read'], 'drivers' )

We are having 3 default permissions which will be created when we run ./manage.py runserver.These permissions will be created for each models in a project.

add: user.has_perm('drivers.add_book')

change: user.has_perm('drivers.change_book')

delete: user.has_perm('drivers.delete_book')

Groups:

With the help of model django.contrib.auth.models.Group,we can categorizing users so you can apply permissions to group(all users)...
For example, if the group Book, author has the permission can_edit_home_page, any user in that group will have that permission.

from myapp.models import Book

from django.contrib.auth.models import Group, Permission

from django.contrib.contenttypes.models import ContentType content_type = ContentType.objects.get_for_model(Book)

permission = Permission.objects.create(codename='can_publish',

                                       name='Can Publish book',

                                       content_type=content_type)

We can directly add a permission to a user using user_permissions and to a group using permissions attribute.

06 Oct 2017 5:00am GMT

• Serializers are used to validate the data in Django rest framework.
• Generally, serializers provide basic validations based on the type of field. Model serializers use model validations(primary key, foreign key, unique, etc).
• We not only use these basic validations but also custom validations to some of the fields.
• For writing custom validations we may require some extra data in validation methods of serializers.
• Django Rest Framework Serializer takes the following parameters while creating the serializer object.

•     read_only, write_only, required, default, initial, source,
      label, help_text, style, error_messages, allow_empty,
      instance, data, partial, context, allow_null
  

• Django serializers designed in such a way that makes an application more powerful and simplified the development process.
• To pass extra context data to the serializer we use "context" parameter.
• "context" parameter takes the data in the form of a dictionary.
• we access the context data inside serializer with statement "self.context"

Let's take an example to understand it more clearly.
Case: Send an email to any user with given content but, do not send email to the specific list of emails.
For this case, we need to validate the email and raise an error if it is in excluded emails list. To do this we need to have excluded emails list in serializers. We can achieve this by using "context" argument of the serializer.
serializers.py

from rest_framework import  serializers

class SendEmailSerializer(serializers.Serializer):
    email = serializers.EmailField()
    content = serializers.CharField(max_length=200)

    def validate_email(self, email):
        # .....
        exclude_email_list = self.context.get("exclude_email_list", [])
        if email in exclude_email_list:
            raise serializers.ValidationError("We cannot send an email to this user")
        # .....
        return email

Rest framework supports both function based views(FBV) and generic views(class-based views CBV).

views.py

# Passing the extra context data to serializers in FBV style.
from rest_framework.decorators import api_view
from your_app.serializers import SendEmailSerializer

@api_view(['POST'])
def send_email_view(request):
    # .....
    context = {"exclude_email_list": ['test@test.com', 'test1@test.com']}
    serializer = SendEmailSerializer(data=request.data, context=context)
    # ....

# Passing the extra context data to serializers in generic CBV or ViewSets style
#ViewSets
from rest_framework import viewsets

class SendEmailViewSet(viewsets.GenericViewSet):
    # ......
    def get_serializer_context(self):
        context = super(CommentViewSet, self).get_serializer_context()
        context.update({
            "exclude_email_list": ['test@test.com', 'test1@test.com']
            # extra data
        })
        return context
    # .......

#Generic Views
from rest_framework import  generics

class SendEmailView(generics.GenericAPIView):
    # ......
    def get_serializer_context(self):
        context = super(CommentViewSet, self).get_serializer_context()
        context.update({
            "exclude_email_list": ['test@test.com', 'test1@test.com']
            # extra data
        })
        return context
    # .......

• In function based views we can pass extra context to serializer with "context" parameter with a dictionary. To access the extra context data inside the serializer we can simply access it with "self.context".
  From example, to get "exclude_email_list" we just used code 'exclude_email_list = self.context.get("exclude_email_list", [])'. context is dictionary and "get" is a dictionary method to access the data.
• It is the best way to override/overload the method get_serializer_context to pass the extra context data to the serializer.
• In viewsets, generic views "get_serializer_context" returns the below data by default.

         {
              'request': self.request,
              'format': self.format_kwarg,
              'view': self
          }
  

• If we want to add extra data to the default we can simply overload it and return the context after updating the context (shown as above)

References:
https://github.com/encode/django-rest-framework/blob/master/rest_framework/serializers.py
https://github.com/encode/django-rest-framework/blob/master/rest_framework/generics.py#L131

05 Oct 2017 5:00am GMT