/*	$NetBSD: pool.h,v 1.64 2008/07/04 16:38:59 ad Exp $	*/

/*-
 * Copyright (c) 1997, 1998, 1999, 2000, 2007 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Paul Kranenburg; by Jason R. Thorpe of the Numerical Aerospace
 * Simulation Facility, NASA Ames Research Center.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef _SYS_POOL_H_
#define _SYS_POOL_H_

#ifdef _KERNEL
#define	__POOL_EXPOSE
#endif

#if defined(_KERNEL_OPT)
#include "opt_pool.h"
#endif

#ifdef __POOL_EXPOSE
#include <sys/mutex.h>
#include <sys/condvar.h>
#include <sys/queue.h>
#include <sys/time.h>
#include <sys/tree.h>
#include <sys/callback.h>
#endif

#define	POOL_PADDR_INVALID	((paddr_t) -1)

#ifdef __POOL_EXPOSE
struct pool;

struct pool_allocator {
	void		*(*pa_alloc)(struct pool *, int);
	void		(*pa_free)(struct pool *, void *);
	unsigned int	pa_pagesz;

	/* The following fields are for internal use only. */
	kmutex_t	pa_lock;
	TAILQ_HEAD(, pool) pa_list;	/* list of pools using this allocator */
	int		pa_flags;
#define	PA_INITIALIZED	0x01
	int		pa_pagemask;
	int		pa_pageshift;
	struct vm_map *pa_backingmap;
#if defined(_KERNEL)
	struct vm_map **pa_backingmapptr;
	SLIST_ENTRY(pool_allocator) pa_q;
#endif /* defined(_KERNEL) */
};

LIST_HEAD(pool_pagelist,pool_item_header);

struct pool {
	TAILQ_ENTRY(pool)
			pr_poollist;
	struct pool_pagelist
			pr_emptypages;	/* Empty pages */
	struct pool_pagelist
			pr_fullpages;	/* Full pages */
	struct pool_pagelist
			pr_partpages;	/* Partially-allocated pages */
	struct pool_item_header	*pr_curpage;
	struct pool	*pr_phpool;	/* Pool item header pool */
	struct pool_cache *pr_cache;	/* Cache for this pool */
	unsigned int	pr_size;	/* Size of item */
	unsigned int	pr_align;	/* Requested alignment, must be 2^n */
	unsigned int	pr_itemoffset;	/* Align this offset in item */
	unsigned int	pr_minitems;	/* minimum # of items to keep */
	unsigned int	pr_minpages;	/* same in page units */
	unsigned int	pr_maxpages;	/* maximum # of pages to keep */
	unsigned int	pr_npages;	/* # of pages allocated */
	unsigned int	pr_itemsperpage;/* # items that fit in a page */
	unsigned int	pr_slack;	/* unused space in a page */
	unsigned int	pr_nitems;	/* number of available items in pool */
	unsigned int	pr_nout;	/* # items currently allocated */
	unsigned int	pr_hardlimit;	/* hard limit to number of allocated
					   items */
	unsigned int	pr_refcnt;	/* ref count for pagedaemon, etc */
	struct pool_allocator *pr_alloc;/* back-end allocator */
	TAILQ_ENTRY(pool) pr_alloc_list;/* link on allocator's pool list */

	/* Drain hook. */
	void		(*pr_drain_hook)(void *, int);
	void		*pr_drain_hook_arg;

	const char	*pr_wchan;	/* tsleep(9) identifier */
	unsigned int	pr_flags;	/* r/w flags */
	unsigned int	pr_roflags;	/* r/o flags */
#define	PR_WAITOK	0x01	/* Note: matches KM_SLEEP */
#define PR_NOWAIT	0x02	/* Note: matches KM_NOSLEEP */
#define PR_WANTED	0x04
#define PR_PHINPAGE	0x40
#define PR_LOGGING	0x80
#define PR_LIMITFAIL	0x100	/* even if waiting, fail if we hit limit */
#define PR_RECURSIVE	0x200	/* pool contains pools, for vmstat(8) */
#define PR_NOTOUCH	0x400	/* don't use free items to keep internal state*/
#define PR_NOALIGN	0x800	/* don't assume backend alignment */
#define	PR_LARGECACHE	0x1000	/* use large cache groups */

	/*
	 * `pr_lock' protects the pool's data structures when removing
	 * items from or returning items to the pool, or when reading
	 * or updating read/write fields in the pool descriptor.
	 *
	 * We assume back-end page allocators provide their own locking
	 * scheme.  They will be called with the pool descriptor _unlocked_,
	 * since the page allocators may block.
	 */
	kmutex_t	pr_lock;
	kcondvar_t	pr_cv;
	int		pr_ipl;

	SPLAY_HEAD(phtree, pool_item_header) pr_phtree;

	int		pr_maxcolor;	/* Cache colouring */
	int		pr_curcolor;
	int		pr_phoffset;	/* Offset in page of page header */

	/*
	 * Warning message to be issued, and a per-time-delta rate cap,
	 * if the hard limit is reached.
	 */
	const char	*pr_hardlimit_warning;
	struct timeval	pr_hardlimit_ratecap;
	struct timeval	pr_hardlimit_warning_last;

	/*
	 * Instrumentation
	 */
	unsigned long	pr_nget;	/* # of successful requests */
	unsigned long	pr_nfail;	/* # of unsuccessful requests */
	unsigned long	pr_nput;	/* # of releases */
	unsigned long	pr_npagealloc;	/* # of pages allocated */
	unsigned long	pr_npagefree;	/* # of pages released */
	unsigned int	pr_hiwat;	/* max # of pages in pool */
	unsigned long	pr_nidle;	/* # of idle pages */

	/*
	 * Diagnostic aides.
	 */
	struct pool_log	*pr_log;
	int		pr_curlogentry;
	int		pr_logsize;

	const char	*pr_entered_file; /* reentrancy check */
	long		pr_entered_line;

	struct callback_entry pr_reclaimerentry;
	void		*pr_freecheck;
	void		*pr_qcache;
};

/*
 * Cache group sizes, assuming 4-byte paddr_t on !_LP64.
 * All groups will be aligned to CACHE_LINE_SIZE.
 */
#ifdef _LP64
#define	PCG_NOBJECTS_NORMAL	15	/* 256 byte group */
#define	PCG_NOBJECTS_LARGE	63	/* 1024 byte group */
#else
#define	PCG_NOBJECTS_NORMAL	14	/* 124 byte group */
#define	PCG_NOBJECTS_LARGE	62	/* 508 byte group */
#endif

typedef struct pcgpair {
	void	*pcgo_va;		/* object virtual address */
	paddr_t	pcgo_pa;		/* object physical address */
} pcgpair_t;

/* The pool cache group. */
typedef struct pool_cache_group {
	struct pool_cache_group	*pcg_next;	/* link to next group */
	u_int			pcg_avail;	/* # available objects */
	u_int			pcg_size;	/* max number objects */
	pcgpair_t 		pcg_objects[1];	/* the objects */
} pcg_t;

typedef struct pool_cache_cpu {
	uint64_t		cc_misses;
	uint64_t		cc_hits;
	struct pool_cache_group	*cc_current;
	struct pool_cache_group	*cc_previous;	
	struct pool_cache	*cc_cache;
	int			cc_ipl;
	int			cc_cpuindex;
#ifdef _KERNEL
	ipl_cookie_t		cc_iplcookie;
#endif
} pool_cache_cpu_t;

struct pool_cache {
	/* Pool layer. */
	struct pool	pc_pool;
	
	/* Cache layer. */
	kmutex_t	pc_lock;	/* locks cache layer */
	TAILQ_ENTRY(pool_cache)
			pc_cachelist;	/* entry on global cache list */
	pcg_t		*pc_emptygroups;/* list of empty cache groups */
	pcg_t		*pc_fullgroups;	/* list of full cache groups */
	pcg_t		*pc_partgroups;	/* groups for reclamation */
	struct pool	*pc_pcgpool;	/* Pool of cache groups */
	int		pc_pcgsize;	/* Use large cache groups? */
	int		pc_ncpu;	/* number cpus set up */
	int		(*pc_ctor)(void *, void *, int);
	void		(*pc_dtor)(void *, void *);
	void		*pc_arg;	/* for ctor/ctor */
	uint64_t	pc_hits;	/* cache layer hits */
	uint64_t	pc_misses;	/* cache layer misses */
	uint64_t	pc_contended;	/* contention events on cache */
	unsigned int	pc_nempty;	/* empty groups in cache */
	unsigned int	pc_nfull;	/* full groups in cache */
	unsigned int	pc_npart;	/* partial groups in cache */
	unsigned int	pc_refcnt;	/* ref count for pagedaemon, etc */
	void		*pc_freecheck;

	/* CPU layer. */
	pool_cache_cpu_t pc_cpu0 __aligned(CACHE_LINE_SIZE);
	void		*pc_cpus[MAXCPUS] __aligned(CACHE_LINE_SIZE);
};

#endif /* __POOL_EXPOSE */

typedef struct pool_cache *pool_cache_t;

#ifdef _KERNEL
/*
 * pool_allocator_kmem is the default that all pools get unless
 * otherwise specified.  pool_allocator_nointr is provided for
 * pools that know they will never be accessed in interrupt
 * context.
 */
extern struct pool_allocator pool_allocator_kmem;
extern struct pool_allocator pool_allocator_nointr;
extern struct pool_allocator pool_allocator_meta;
#ifdef POOL_SUBPAGE
/* The above are subpage allocators in this case. */
extern struct pool_allocator pool_allocator_kmem_fullpage;
extern struct pool_allocator pool_allocator_nointr_fullpage;
#endif

struct link_pool_init {	/* same as args to pool_init() */
	struct pool *pp;
	size_t size;
	u_int align;
	u_int align_offset;
	int flags;
	const char *wchan;
	struct pool_allocator *palloc;
	int ipl;
};
#define	POOL_INIT(pp, size, align, align_offset, flags, wchan, palloc, ipl)\
struct pool pp;								\
static const struct link_pool_init _link_ ## pp[1] = {			\
	{ &pp, size, align, align_offset, flags, wchan, palloc, ipl }	\
};									\
__link_set_add_rodata(pools, _link_ ## pp)

void		pool_subsystem_init(void);

void		pool_init(struct pool *, size_t, u_int, u_int,
		    int, const char *, struct pool_allocator *, int);
void		pool_destroy(struct pool *);

void		pool_set_drain_hook(struct pool *,
		    void (*)(void *, int), void *);

void		*pool_get(struct pool *, int);
void		pool_put(struct pool *, void *);
int		pool_reclaim(struct pool *);

#ifdef POOL_DIAGNOSTIC
/*
 * These versions do reentrancy checking.
 */
void		*_pool_get(struct pool *, int, const char *, long);
void		_pool_put(struct pool *, void *, const char *, long);
int		_pool_reclaim(struct pool *, const char *, long);
#define		pool_get(h, f)	_pool_get((h), (f), __FILE__, __LINE__)
#define		pool_put(h, v)	_pool_put((h), (v), __FILE__, __LINE__)
#define		pool_reclaim(h)	_pool_reclaim((h), __FILE__, __LINE__)
#endif /* POOL_DIAGNOSTIC */

int		pool_prime(struct pool *, int);
void		pool_setlowat(struct pool *, int);
void		pool_sethiwat(struct pool *, int);
void		pool_sethardlimit(struct pool *, int, const char *, int);
void		pool_drain_start(struct pool **, uint64_t *);
void		pool_drain_end(struct pool *, uint64_t);

/*
 * Debugging and diagnostic aides.
 */
void		pool_print(struct pool *, const char *);
void		pool_printit(struct pool *, const char *,
		    void (*)(const char *, ...));
void		pool_printall(const char *, void (*)(const char *, ...));
int		pool_chk(struct pool *, const char *);

/*
 * Pool cache routines.
 */
pool_cache_t	pool_cache_init(size_t, u_int, u_int, u_int, const char *,
		    struct pool_allocator *, int, int (*)(void *, void *, int),
		    void (*)(void *, void *), void *);
void		pool_cache_bootstrap(pool_cache_t, size_t, u_int, u_int, u_int,
		    const char *, struct pool_allocator *, int,
		    int (*)(void *, void *, int), void (*)(void *, void *),
		    void *);
void		pool_cache_destroy(pool_cache_t);
void		*pool_cache_get_paddr(pool_cache_t, int, paddr_t *);
void		pool_cache_put_paddr(pool_cache_t, void *, paddr_t);
void		pool_cache_destruct_object(pool_cache_t, void *);
void		pool_cache_invalidate(pool_cache_t);
bool		pool_cache_reclaim(pool_cache_t);
void		pool_cache_set_drain_hook(pool_cache_t,
		    void (*)(void *, int), void *);
void		pool_cache_setlowat(pool_cache_t, int);
void		pool_cache_sethiwat(pool_cache_t, int);
void		pool_cache_sethardlimit(pool_cache_t, int, const char *, int);
void		pool_cache_cpu_init(struct cpu_info *);

#define		pool_cache_get(pc, f) pool_cache_get_paddr((pc), (f), NULL)
#define		pool_cache_put(pc, o) pool_cache_put_paddr((pc), (o), \
				          POOL_PADDR_INVALID)

void 		pool_whatis(uintptr_t, void (*)(const char *, ...));
#endif /* _KERNEL */

#endif /* _SYS_POOL_H_ */
