| 35,6 → 35,7 |
|---|
| #include <linux/spinlock.h> |
| //#include <linux/preempt.h> |
| #include <linux/lockdep.h> |
| #include <linux/compiler.h> |
| #include <asm/processor.h> |
| |
| /* |
| 108,7 → 109,7 |
|---|
| unsigned ret; |
| |
| repeat: |
| ret = ACCESS_ONCE(s->sequence); |
| ret = READ_ONCE(s->sequence); |
| if (unlikely(ret & 1)) { |
| cpu_relax(); |
| goto repeat; |
| 127,7 → 128,7 |
|---|
| */ |
| static inline unsigned raw_read_seqcount(const seqcount_t *s) |
| { |
| unsigned ret = ACCESS_ONCE(s->sequence); |
| unsigned ret = READ_ONCE(s->sequence); |
| smp_rmb(); |
| return ret; |
| } |
| 179,7 → 180,7 |
|---|
| */ |
| static inline unsigned raw_seqcount_begin(const seqcount_t *s) |
| { |
| unsigned ret = ACCESS_ONCE(s->sequence); |
| unsigned ret = READ_ONCE(s->sequence); |
| smp_rmb(); |
| return ret & ~1; |
| } |
| 236,6 → 237,79 |
|---|
| /* |
| * raw_write_seqcount_latch - redirect readers to even/odd copy |
| * @s: pointer to seqcount_t |
| * |
| * The latch technique is a multiversion concurrency control method that allows |
| * queries during non-atomic modifications. If you can guarantee queries never |
| * interrupt the modification -- e.g. the concurrency is strictly between CPUs |
| * -- you most likely do not need this. |
| * |
| * Where the traditional RCU/lockless data structures rely on atomic |
| * modifications to ensure queries observe either the old or the new state the |
| * latch allows the same for non-atomic updates. The trade-off is doubling the |
| * cost of storage; we have to maintain two copies of the entire data |
| * structure. |
| * |
| * Very simply put: we first modify one copy and then the other. This ensures |
| * there is always one copy in a stable state, ready to give us an answer. |
| * |
| * The basic form is a data structure like: |
| * |
| * struct latch_struct { |
| * seqcount_t seq; |
| * struct data_struct data[2]; |
| * }; |
| * |
| * Where a modification, which is assumed to be externally serialized, does the |
| * following: |
| * |
| * void latch_modify(struct latch_struct *latch, ...) |
| * { |
| * smp_wmb(); <- Ensure that the last data[1] update is visible |
| * latch->seq++; |
| * smp_wmb(); <- Ensure that the seqcount update is visible |
| * |
| * modify(latch->data[0], ...); |
| * |
| * smp_wmb(); <- Ensure that the data[0] update is visible |
| * latch->seq++; |
| * smp_wmb(); <- Ensure that the seqcount update is visible |
| * |
| * modify(latch->data[1], ...); |
| * } |
| * |
| * The query will have a form like: |
| * |
| * struct entry *latch_query(struct latch_struct *latch, ...) |
| * { |
| * struct entry *entry; |
| * unsigned seq, idx; |
| * |
| * do { |
| * seq = lockless_dereference(latch->seq); |
| * |
| * idx = seq & 0x01; |
| * entry = data_query(latch->data[idx], ...); |
| * |
| * smp_rmb(); |
| * } while (seq != latch->seq); |
| * |
| * return entry; |
| * } |
| * |
| * So during the modification, queries are first redirected to data[1]. Then we |
| * modify data[0]. When that is complete, we redirect queries back to data[0] |
| * and we can modify data[1]. |
| * |
| * NOTE: The non-requirement for atomic modifications does _NOT_ include |
| * the publishing of new entries in the case where data is a dynamic |
| * data structure. |
| * |
| * An iteration might start in data[0] and get suspended long enough |
| * to miss an entire modification sequence, once it resumes it might |
| * observe the new entry. |
| * |
| * NOTE: When data is a dynamic data structure; one should use regular RCU |
| * patterns to manage the lifetimes of the objects within. |
| */ |
| static inline void raw_write_seqcount_latch(seqcount_t *s) |
| { |
| 266,13 → 340,13 |
|---|
| } |
| |
| /** |
| * write_seqcount_barrier - invalidate in-progress read-side seq operations |
| * write_seqcount_invalidate - invalidate in-progress read-side seq operations |
| * @s: pointer to seqcount_t |
| * |
| * After write_seqcount_barrier, no read-side seq operations will complete |
| * After write_seqcount_invalidate, no read-side seq operations will complete |
| * successfully and see data older than this. |
| */ |
| static inline void write_seqcount_barrier(seqcount_t *s) |
| static inline void write_seqcount_invalidate(seqcount_t *s) |
| { |
| smp_wmb(); |
| s->sequence+=2; |