// SPDX-License-Identifier: GPL-2.0-or-later

  /*
   * shadow.c - Shadow Variables
   *
   * Copyright (C) 2014 Josh Poimboeuf <jpoimboe@redhat.com>
   * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
   * Copyright (C) 2017 Joe Lawrence <joe.lawrence@redhat.com>

   */
  
  /**
   * DOC: Shadow variable API concurrency notes:
   *
   * The shadow variable API provides a simple relationship between an
   * <obj, id> pair and a pointer value.  It is the responsibility of the
   * caller to provide any mutual exclusion required of the shadow data.
   *
   * Once a shadow variable is attached to its parent object via the
   * klp_shadow_*alloc() API calls, it is considered live: any subsequent
   * call to klp_shadow_get() may then return the shadow variable's data
   * pointer.  Callers of klp_shadow_*alloc() should prepare shadow data
   * accordingly.
   *
   * The klp_shadow_*alloc() API calls may allocate memory for new shadow
   * variable structures.  Their implementation does not call kmalloc
   * inside any spinlocks, but API callers should pass GFP flags according
   * to their specific needs.
   *
   * The klp_shadow_hash is an RCU-enabled hashtable and is safe against
   * concurrent klp_shadow_free() and klp_shadow_get() operations.
   */
  
  #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  
  #include <linux/hashtable.h>
  #include <linux/slab.h>
  #include <linux/livepatch.h>
  
  static DEFINE_HASHTABLE(klp_shadow_hash, 12);
  
  /*
   * klp_shadow_lock provides exclusive access to the klp_shadow_hash and
   * the shadow variables it references.
   */
  static DEFINE_SPINLOCK(klp_shadow_lock);
  
  /**
   * struct klp_shadow - shadow variable structure
   * @node:	klp_shadow_hash hash table node
   * @rcu_head:	RCU is used to safely free this structure
   * @obj:	pointer to parent object
   * @id:		data identifier
   * @data:	data area
   */
  struct klp_shadow {
  	struct hlist_node node;
  	struct rcu_head rcu_head;
  	void *obj;
  	unsigned long id;
  	char data[];
  };
  
  /**
   * klp_shadow_match() - verify a shadow variable matches given <obj, id>
   * @shadow:	shadow variable to match
   * @obj:	pointer to parent object
   * @id:		data identifier
   *
   * Return: true if the shadow variable matches.
   */
  static inline bool klp_shadow_match(struct klp_shadow *shadow, void *obj,
  				unsigned long id)
  {
  	return shadow->obj == obj && shadow->id == id;
  }
  
  /**
   * klp_shadow_get() - retrieve a shadow variable data pointer
   * @obj:	pointer to parent object
   * @id:		data identifier
   *
   * Return: the shadow variable data element, NULL on failure.
   */
  void *klp_shadow_get(void *obj, unsigned long id)
  {
  	struct klp_shadow *shadow;
  
  	rcu_read_lock();
  
  	hash_for_each_possible_rcu(klp_shadow_hash, shadow, node,
  				   (unsigned long)obj) {
  
  		if (klp_shadow_match(shadow, obj, id)) {
  			rcu_read_unlock();
  			return shadow->data;
  		}
  	}
  
  	rcu_read_unlock();
  
  	return NULL;
  }
  EXPORT_SYMBOL_GPL(klp_shadow_get);

  static void *__klp_shadow_get_or_alloc(void *obj, unsigned long id,
  				       size_t size, gfp_t gfp_flags,
  				       klp_shadow_ctor_t ctor, void *ctor_data,
  				       bool warn_on_exist)

  {
  	struct klp_shadow *new_shadow;
  	void *shadow_data;
  	unsigned long flags;
  
  	/* Check if the shadow variable already exists */
  	shadow_data = klp_shadow_get(obj, id);
  	if (shadow_data)
  		goto exists;

  	/*
  	 * Allocate a new shadow variable.  Fill it with zeroes by default.
  	 * More complex setting can be done by @ctor function.  But it is
  	 * called only when the buffer is really used (under klp_shadow_lock).
  	 */

  	new_shadow = kzalloc(size + sizeof(*new_shadow), gfp_flags);
  	if (!new_shadow)
  		return NULL;

  	/* Look for <obj, id> again under the lock */
  	spin_lock_irqsave(&klp_shadow_lock, flags);
  	shadow_data = klp_shadow_get(obj, id);
  	if (unlikely(shadow_data)) {
  		/*
  		 * Shadow variable was found, throw away speculative
  		 * allocation.
  		 */
  		spin_unlock_irqrestore(&klp_shadow_lock, flags);
  		kfree(new_shadow);
  		goto exists;
  	}

  	new_shadow->obj = obj;
  	new_shadow->id = id;
  
  	if (ctor) {
  		int err;
  
  		err = ctor(obj, new_shadow->data, ctor_data);
  		if (err) {
  			spin_unlock_irqrestore(&klp_shadow_lock, flags);
  			kfree(new_shadow);
  			pr_err("Failed to construct shadow variable <%p, %lx> (%d)
  ",
  			       obj, id, err);
  			return NULL;
  		}
  	}

  	/* No <obj, id> found, so attach the newly allocated one */
  	hash_add_rcu(klp_shadow_hash, &new_shadow->node,
  		     (unsigned long)new_shadow->obj);
  	spin_unlock_irqrestore(&klp_shadow_lock, flags);
  
  	return new_shadow->data;
  
  exists:
  	if (warn_on_exist) {
  		WARN(1, "Duplicate shadow variable <%p, %lx>
  ", obj, id);
  		return NULL;
  	}
  
  	return shadow_data;
  }
  
  /**
   * klp_shadow_alloc() - allocate and add a new shadow variable
   * @obj:	pointer to parent object
   * @id:		data identifier

   * @size:	size of attached data
   * @gfp_flags:	GFP mask for allocation

   * @ctor:	custom constructor to initialize the shadow data (optional)
   * @ctor_data:	pointer to any data needed by @ctor (optional)
   *
   * Allocates @size bytes for new shadow variable data using @gfp_flags.
   * The data are zeroed by default.  They are further initialized by @ctor
   * function if it is not NULL.  The new shadow variable is then added
   * to the global hashtable.

   *

   * If an existing <obj, id> shadow variable can be found, this routine will
   * issue a WARN, exit early and return NULL.

   *

   * This function guarantees that the constructor function is called only when
   * the variable did not exist before.  The cost is that @ctor is called
   * in atomic context under a spin lock.

   *
   * Return: the shadow variable data element, NULL on duplicate or
   * failure.
   */

  void *klp_shadow_alloc(void *obj, unsigned long id,
  		       size_t size, gfp_t gfp_flags,
  		       klp_shadow_ctor_t ctor, void *ctor_data)

  {

  	return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
  					 ctor, ctor_data, true);

  }
  EXPORT_SYMBOL_GPL(klp_shadow_alloc);
  
  /**
   * klp_shadow_get_or_alloc() - get existing or allocate a new shadow variable
   * @obj:	pointer to parent object
   * @id:		data identifier

   * @size:	size of attached data
   * @gfp_flags:	GFP mask for allocation

   * @ctor:	custom constructor to initialize the shadow data (optional)
   * @ctor_data:	pointer to any data needed by @ctor (optional)

   *
   * Returns a pointer to existing shadow data if an <obj, id> shadow
   * variable is already present.  Otherwise, it creates a new shadow
   * variable like klp_shadow_alloc().
   *

   * This function guarantees that only one shadow variable exists with the given
   * @id for the given @obj.  It also guarantees that the constructor function
   * will be called only when the variable did not exist before.  The cost is
   * that @ctor is called in atomic context under a spin lock.

   *
   * Return: the shadow variable data element, NULL on failure.
   */

  void *klp_shadow_get_or_alloc(void *obj, unsigned long id,
  			      size_t size, gfp_t gfp_flags,
  			      klp_shadow_ctor_t ctor, void *ctor_data)

  {

  	return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
  					 ctor, ctor_data, false);

  }
  EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc);

  static void klp_shadow_free_struct(struct klp_shadow *shadow,
  				   klp_shadow_dtor_t dtor)
  {
  	hash_del_rcu(&shadow->node);
  	if (dtor)
  		dtor(shadow->obj, shadow->data);
  	kfree_rcu(shadow, rcu_head);
  }

  /**
   * klp_shadow_free() - detach and free a <obj, id> shadow variable
   * @obj:	pointer to parent object
   * @id:		data identifier

   * @dtor:	custom callback that can be used to unregister the variable
   *		and/or free data that the shadow variable points to (optional)

   *
   * This function releases the memory for this <obj, id> shadow variable
   * instance, callers should stop referencing it accordingly.
   */

  void klp_shadow_free(void *obj, unsigned long id, klp_shadow_dtor_t dtor)

  {
  	struct klp_shadow *shadow;
  	unsigned long flags;
  
  	spin_lock_irqsave(&klp_shadow_lock, flags);
  
  	/* Delete <obj, id> from hash */
  	hash_for_each_possible(klp_shadow_hash, shadow, node,
  			       (unsigned long)obj) {
  
  		if (klp_shadow_match(shadow, obj, id)) {

  			klp_shadow_free_struct(shadow, dtor);

  			break;
  		}
  	}
  
  	spin_unlock_irqrestore(&klp_shadow_lock, flags);
  }
  EXPORT_SYMBOL_GPL(klp_shadow_free);
  
  /**
   * klp_shadow_free_all() - detach and free all <*, id> shadow variables
   * @id:		data identifier

   * @dtor:	custom callback that can be used to unregister the variable
   *		and/or free data that the shadow variable points to (optional)

   *
   * This function releases the memory for all <*, id> shadow variable
   * instances, callers should stop referencing them accordingly.
   */

  void klp_shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor)

  {
  	struct klp_shadow *shadow;
  	unsigned long flags;
  	int i;
  
  	spin_lock_irqsave(&klp_shadow_lock, flags);
  
  	/* Delete all <*, id> from hash */
  	hash_for_each(klp_shadow_hash, i, shadow, node) {

  		if (klp_shadow_match(shadow, shadow->obj, id))
  			klp_shadow_free_struct(shadow, dtor);

  	}
  
  	spin_unlock_irqrestore(&klp_shadow_lock, flags);
  }
  EXPORT_SYMBOL_GPL(klp_shadow_free_all);